feat(webui): Add log-ingestor and S3 auth config to deployment templates.

components/clp-package-utils/clp_package_utils/controller.py

@@ -794,12 +794,27 @@ def _set_up_env_for_webui(self, container_clp_config: ClpConfig) -> EnvVarsDict:
             server_settings_json_updates["PrestoHost"] = None
             server_settings_json_updates["PrestoPort"] = None
 
+        if (
+            self._clp_config.log_ingestor is not None
+            and self._clp_config.logs_input.type == StorageType.S3
+        ):
+            server_settings_json_updates["LogIngestorHost"] = container_clp_config.log_ingestor.host
+            server_settings_json_updates["LogIngestorPort"] = container_clp_config.log_ingestor.port
+        else:
+            server_settings_json_updates["LogIngestorHost"] = None
+            server_settings_json_updates["LogIngestorPort"] = None
+
         if StorageType.FS == self._clp_config.logs_input.type:
             client_settings_json_updates["LogsInputRootDir"] = str(CONTAINER_INPUT_LOGS_ROOT_DIR)
             server_settings_json_updates["LogsInputRootDir"] = str(CONTAINER_INPUT_LOGS_ROOT_DIR)
+            server_settings_json_updates["LogsInputS3AwsAuthType"] = None
+            server_settings_json_updates["LogsInputS3AwsProfile"] = None
         else:
             client_settings_json_updates["LogsInputRootDir"] = None
             server_settings_json_updates["LogsInputRootDir"] = None
+            s3_auth = self._clp_config.logs_input.aws_authentication
+            server_settings_json_updates["LogsInputS3AwsAuthType"] = s3_auth.type
+            server_settings_json_updates["LogsInputS3AwsProfile"] = s3_auth.profile
 
         resolved_client_settings_json_path = resolve_host_path_in_container(
             client_settings_json_path
@@ -1000,6 +1015,18 @@ def set_up_env(self) -> None:
                     ),
                 }
 
+        if self._clp_config.logs_input.type == StorageType.S3:
+            logs_input_aws_auth = self._clp_config.logs_input.aws_authentication
+            if logs_input_aws_auth.type == AwsAuthType.credentials:
+                env_vars |= {
+                    "CLP_LOGS_INPUT_AWS_ACCESS_KEY_ID": (
+                        logs_input_aws_auth.credentials.access_key_id
+                    ),
+                    "CLP_LOGS_INPUT_AWS_SECRET_ACCESS_KEY": (
+                        logs_input_aws_auth.credentials.secret_access_key
+                    ),
+                }
+
         # Identity config
         env_vars |= {
             "CLP_FIRST_PARTY_SERVICE_UID_GID": DEFAULT_UID_GID,

components/clp-py-utils/clp_py_utils/clp_config.py

@@ -777,10 +777,16 @@ class ApiServer(BaseModel):
 
 
 class LogIngestor(BaseModel):
+    DEFAULT_PORT: ClassVar[int] = 3002
+
     host: DomainStr = "localhost"
-    port: Port = 3002
+    port: Port = DEFAULT_PORT
     logging_level: LoggingLevelRust = "INFO"
 
+    def transform_for_container(self):
+        self.host = LOG_INGESTOR_COMPONENT_NAME
+        self.port = self.DEFAULT_PORT
+
 
 class Presto(BaseModel):
     DEFAULT_PORT: ClassVar[int] = 8080
@@ -1092,6 +1098,8 @@ def transform_for_container(self):
         self.results_cache.transform_for_container(BundledService.RESULTS_CACHE in self.bundled)
         self.query_scheduler.transform_for_container()
         self.reducer.transform_for_container()
+        if self.log_ingestor is not None:
+            self.log_ingestor.transform_for_container()
         if self.package.query_engine == QueryEngine.PRESTO and self.presto is not None:
             self.presto.transform_for_container()
 

components/webui/server/.env

@@ -18,3 +18,7 @@ PRESTO_SCHEMA=default
 # Security
 RATE_LIMIT=1000
 
+# S3
+CLP_LOGS_INPUT_AWS_ACCESS_KEY_ID=
+CLP_LOGS_INPUT_AWS_SECRET_ACCESS_KEY=
+

components/webui/server/src/plugins/external/env.ts

@@ -13,6 +13,8 @@ declare module "fastify" {
             PRESTO_CATALOG: string;
             PRESTO_SCHEMA: string;
             RATE_LIMIT: number;
+            CLP_LOGS_INPUT_AWS_ACCESS_KEY_ID: string;
+            CLP_LOGS_INPUT_AWS_SECRET_ACCESS_KEY: string;
         };
     }
 }
@@ -72,6 +74,16 @@ const schema = {
             default: 1_000,
             minimum: 1,
         },
+
+        // S3
+        CLP_LOGS_INPUT_AWS_ACCESS_KEY_ID: {
+            type: "string",
+            default: "",
+        },
+        CLP_LOGS_INPUT_AWS_SECRET_ACCESS_KEY: {
+            type: "string",
+            default: "",
+        },
     },
 };
 

tools/deployment/package-helm/templates/configmap.yaml

@@ -284,10 +284,28 @@ data:
         {{ .Values.clpConfig.results_cache.stream_collection_name | quote }},
       "ClientDir": "/opt/clp/var/www/webui/client",
       "LogViewerDir": "/opt/clp/var/www/webui/yscope-log-viewer",
+      {{- if and .Values.clpConfig.log_ingestor (eq .Values.clpConfig.logs_input.type "s3") }}
+      "LogIngestorHost": "{{ include "clp.fullname" . }}-log-ingestor",
+      "LogIngestorPort": 3002,
+      {{- else }}
+      "LogIngestorHost": null,
+      "LogIngestorPort": null,
+      {{- end }}{{/* if and .Values.clpConfig.log_ingestor
+                          (eq .Values.clpConfig.logs_input.type "s3") */}}
       {{- if eq .Values.clpConfig.logs_input.type "fs" }}
       "LogsInputRootDir": "/mnt/logs",
+      "LogsInputS3AwsAuthType": null,
+      "LogsInputS3AwsProfile": null,
       {{- else }}
       "LogsInputRootDir": null,
+      {{- with .Values.clpConfig.logs_input.aws_authentication }}
+      "LogsInputS3AwsAuthType": {{ .type | quote }},
+      {{- if .profile }}
+      "LogsInputS3AwsProfile": {{ .profile | quote }},
+      {{- else }}
+      "LogsInputS3AwsProfile": null,
+      {{- end }}{{/* if .profile */}}
+      {{- end }}{{/* with .Values.clpConfig.logs_input.aws_authentication */}}
       {{- end }}
       {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }}
       "StreamFilesDir": "/var/data/streams",

tools/deployment/package-helm/templates/webui-deployment.yaml

@@ -68,6 +68,15 @@ spec:
             {{- end }}{{/* if and (eq .type "s3")
                                   (eq .s3_config.aws_authentication.type "credentials") */}}
             {{- end }}{{/* with .Values.clpConfig.stream_output.storage */}}
+            {{- with .Values.clpConfig.logs_input }}
+            {{- if and (eq .type "s3") (eq .aws_authentication.type "credentials") }}
+            - name: "CLP_LOGS_INPUT_AWS_ACCESS_KEY_ID"
+              value: {{ .aws_authentication.credentials.access_key_id | quote }}
+            - name: "CLP_LOGS_INPUT_AWS_SECRET_ACCESS_KEY"
+              value: {{ .aws_authentication.credentials.secret_access_key | quote }}
+            {{- end }}{{/* if and (eq .type "s3")
+                                  (eq .aws_authentication.type "credentials") */}}
+            {{- end }}{{/* with .Values.clpConfig.logs_input */}}
           ports:
             - name: "webui"
               containerPort: 4000

tools/deployment/package/docker-compose-all.yaml

@@ -370,6 +370,8 @@ services:
       AWS_SECRET_ACCESS_KEY: "${CLP_STREAM_OUTPUT_AWS_SECRET_ACCESS_KEY:-}"
       CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}"
       CLP_DB_USER: "${CLP_DB_USER:-clp-user}"
+      CLP_LOGS_INPUT_AWS_ACCESS_KEY_ID: "${CLP_LOGS_INPUT_AWS_ACCESS_KEY_ID:-}"
+      CLP_LOGS_INPUT_AWS_SECRET_ACCESS_KEY: "${CLP_LOGS_INPUT_AWS_SECRET_ACCESS_KEY:-}"
       HOST: "0.0.0.0"
       NODE_ENV: "production"
       NODE_PATH: "/opt/clp/var/www/webui/server/node_modules"