Merge branch 'main' into aws-hostpath

Author: junhaoliao

.gitattributes

@@ -0,0 +1 @@
+* text=auto eol=lf

.github/actions/clp-build-runtime-image/action.yaml

@@ -21,6 +21,10 @@ inputs:
     description: "Platform VERSION_ID / VERSION_CODENAME of the container
       (e.g. jammy, focal, etc.)"
     required: false
+  arch:
+    description: "Target architecture (amd64 or arm64)"
+    default: "amd64"
+    required: false
 
 runs:
   using: "composite"
@@ -29,7 +33,7 @@ runs:
       uses: "docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2"
 
     - name: "Login to Image Registry"
-      uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772"
+      uses: "docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121"  # v4.1.0
       with:
         registry: "${{inputs.image_registry}}"
         username: "${{inputs.image_registry_username}}"
@@ -61,13 +65,32 @@ runs:
       with:
         images: "${{inputs.image_registry}}/${{steps.sanitization.outputs.REPOSITORY}}\
           /${{steps.compute-meta.outputs.IMAGE_NAME}}"
+        flavor: |
+          suffix=-${{inputs.arch}}
+
+    - name: "Map arch to Docker platform"
+      id: "map-arch"
+      shell: "bash"
+      run: |
+        if [ "${{inputs.arch}}" = "arm64" ]; then
+          echo "platform=linux/arm64" >> "$GITHUB_OUTPUT"
+        elif [ "${{inputs.arch}}" = "amd64" ]; then
+          echo "platform=linux/amd64" >> "$GITHUB_OUTPUT"
+        else
+          echo >&2 "ERROR: Unsupported arch '${{inputs.arch}}'. Must be 'amd64' or 'arm64'."
+          exit 1
+        fi
 
     - name: "Build and Push"
       if: "github.event_name != 'pull_request' && github.ref == 'refs/heads/main'"
       uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4"
       with:
         context: "./"
         file: "${{steps.compute-meta.outputs.DOCKERFILE}}"
+        platforms: "${{steps.map-arch.outputs.platform}}"
+        # Disable provenance to create a simple image instead of a manifest list.
+        # This allows `docker manifest create` to combine the per-arch images later.
+        provenance: false
         push: true
         tags: "${{steps.extract-gh-meta.outputs.tags}}"
         labels: "${{steps.extract-gh-meta.outputs.labels}}"

.github/workflows/clp-artifact-build.yaml

@@ -250,8 +250,8 @@ jobs:
             ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}}
           token: "${{secrets.GITHUB_TOKEN}}"
 
-  ubuntu-jammy-deps-image:
-    name: "ubuntu-jammy-deps-image"
+  ubuntu-jammy-x86_64-deps-image:
+    name: "ubuntu-jammy-x86_64-deps-image"
     if: "needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'true'"
     needs: "filter-relevant-changes"
     runs-on: *runner
@@ -276,6 +276,35 @@ jobs:
             ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}}
           token: "${{secrets.GITHUB_TOKEN}}"
 
+  ubuntu-jammy-aarch64-deps-image:
+    name: "ubuntu-jammy-aarch64-deps-image"
+    if: >-
+      needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'true'
+      && github.event_name != 'pull_request'
+      && github.ref == 'refs/heads/main'
+    needs: "filter-relevant-changes"
+    runs-on: "ubuntu-24.04-arm"
+    steps:
+      - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd"  # v6.0.2
+        with:
+          submodules: "recursive"
+
+      - name: "Work around actions/runner-images/issues/6775"
+        run: "chown $(id -u):$(id -g) -R ."
+        shell: "bash"
+
+      - uses: "./.github/actions/clp-core-build-containers"
+        env:
+          OS_NAME: "ubuntu-jammy"
+        with:
+          image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_AARCH64}}${{env.OS_NAME}}"
+          docker_context: "components/core"
+          docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\
+            /Dockerfile"
+          push_deps_image: >-
+            ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}}
+          token: "${{secrets.GITHUB_TOKEN}}"
+
   centos-stream-9-binaries:
     # Run if the ancestor jobs succeeded OR they were skipped and clp was changed.
     if: >-
@@ -455,7 +484,7 @@ jobs:
       || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true')
     needs:
       - "filter-relevant-changes"
-      - "ubuntu-jammy-deps-image"
+      - "ubuntu-jammy-x86_64-deps-image"
     strategy:
       matrix:
         include:
@@ -539,7 +568,7 @@ jobs:
           tar xf clp.tar
           rm clp.tar
 
-      - uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772"
+      - uses: "docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121"  # v4.1.0
         with:
           registry: "ghcr.io"
           username: "${{github.actor}}"
@@ -580,7 +609,7 @@ jobs:
       || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true')
     needs:
       - "filter-relevant-changes"
-      - "ubuntu-jammy-deps-image"
+      - "ubuntu-jammy-x86_64-deps-image"
     runs-on: *runner
     steps:
       - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd"  # v6.0.2
@@ -649,16 +678,35 @@ jobs:
             )}}
 
   package-image:
-    name: "package-image"
+    name: "package-image-${{matrix.arch}}"
+    # NOTE: The aarch64 job is skipped on PRs, so we accept both 'success' and 'skipped'.
     if: >-
       !cancelled() && !failure() && (
         needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' ||
-        needs.ubuntu-jammy-deps-image.result == 'success'
+        (needs.ubuntu-jammy-x86_64-deps-image.result == 'success'
+        && (needs.ubuntu-jammy-aarch64-deps-image.result == 'success'
+        || needs.ubuntu-jammy-aarch64-deps-image.result == 'skipped'))
       )
     needs:
       - "filter-relevant-changes"
-      - "ubuntu-jammy-deps-image"
-    runs-on: *runner
+      - "ubuntu-jammy-x86_64-deps-image"
+      - "ubuntu-jammy-aarch64-deps-image"
+    strategy:
+      matrix:
+        # Only build arm64 images on push to main to save CI resources on PRs.
+        arch: >-
+          ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
+          && fromJSON('["amd64", "arm64"]') || fromJSON('["amd64"]')}}
+    runs-on: >-
+      ${{
+        matrix.arch == 'amd64'
+        && (github.repository_owner == 'y-scope'
+            && fromJSON('["self-hosted", "x64", "ubuntu-noble"]')
+            || 'ubuntu-24.04')
+        || (github.repository_owner == 'y-scope'
+            && fromJSON('["self-hosted", "arm64", "ubuntu-noble"]')
+            || 'ubuntu-24.04-arm')
+      }}
     steps:
       - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd"  # v6.0.2
         with:
@@ -673,7 +721,10 @@ jobs:
         env:
           OS_NAME: "ubuntu-jammy"
         with:
-          image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}"
+          image_name: >-
+            ${{format('{0}ubuntu-jammy',
+            matrix.arch == 'amd64' && env.DEPS_IMAGE_NAME_PREFIX_X86
+            || env.DEPS_IMAGE_NAME_PREFIX_AARCH64)}}
           use_published_image: >-
             ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false'
             || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}}
@@ -687,3 +738,42 @@ jobs:
           image_registry: "ghcr.io"
           image_registry_username: "${{github.actor}}"
           image_registry_password: "${{secrets.GITHUB_TOKEN}}"
+          arch: "${{matrix.arch}}"
+
+  package-image-multiarch-manifest:
+    name: "package-image-multiarch-manifest"
+    if: >-
+      github.event_name != 'pull_request'
+      && github.ref == 'refs/heads/main'
+      && needs.package-image.result == 'success'
+    needs: "package-image"
+    runs-on: *runner
+    steps:
+      - name: "Login to Image Registry"
+        uses: "docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121"  # v4.1.0
+        with:
+          registry: "ghcr.io"
+          username: "${{github.actor}}"
+          password: "${{secrets.GITHUB_TOKEN}}"
+
+      - name: "Sanitize Repository Name"
+        id: "sanitization"
+        shell: "bash"
+        run: |
+          echo "REPOSITORY=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]')" \
+            >> "$GITHUB_OUTPUT"
+
+      - name: "Create and Push Multi-arch Manifest"
+        shell: "bash"
+        run: |
+          image_base="ghcr.io/${{steps.sanitization.outputs.REPOSITORY}}/clp-package"
+          tags=("${{github.ref_name}}")
+          if [ "${{github.event_name}}" = "schedule" ]; then
+            tags+=("nightly")
+          fi
+          for tag in "${tags[@]}"; do
+            docker manifest create "${image_base}:${tag}" \
+              "${image_base}:${tag}-amd64" \
+              "${image_base}:${tag}-arm64"
+            docker manifest push "${image_base}:${tag}"
+          done

components/clp-package-utils/clp_package_utils/controller.py

@@ -168,9 +168,9 @@ def _set_up_env_for_database_bundling(self) -> EnvVarsDict:
         # Runtime config
         env_vars |= {
             "CLP_DB_CONTAINER_IMAGE_REF": (
-                "mysql:8.0.23"
+                "mysql:8.0.46"
                 if self._clp_config.database.type == DatabaseEngine.MYSQL
-                else "mariadb:10-jammy"
+                else "mariadb:10.11.16"
             ),
         }
 

components/clp-rust-utils/src/job_config/ingestion.rs

@@ -28,6 +28,14 @@ pub mod s3 {
                 Self::S3Scanner(config) => &config.base,
             }
         }
+
+        #[must_use]
+        pub const fn as_buffer_config(&self) -> &BufferConfig {
+            match self {
+                Self::SqsListener(config) => &config.buffer_config,
+                Self::S3Scanner(config) => &config.buffer_config,
+            }
+        }
     }
 
     /// Base configuration for ingesting logs from S3.
@@ -65,10 +73,6 @@ pub mod s3 {
         /// Whether to treat the ingested objects as unstructured logs. Defaults to `false`.
         #[serde(default = "default_unstructured")]
         pub unstructured: bool,
-
-        /// Per-job ingestion buffer config.
-        #[serde(default)]
-        pub buffer_config: BufferConfig,
     }
 
     /// Configuration for a SQS listener job.
@@ -77,6 +81,10 @@ pub mod s3 {
         #[serde(flatten)]
         pub base: BaseConfig,
 
+        /// Per-job ingestion buffer config.
+        #[serde(default)]
+        pub buffer_config: BufferConfig,
+
         /// The SQS queue URL to poll for S3 event notifications. The given queue must be dedicated
         /// to this ingestion job.
         #[schema(value_type = String, min_length = 1)]
@@ -153,6 +161,10 @@ pub mod s3 {
         #[serde(flatten)]
         pub base: BaseConfig,
 
+        /// Per-job ingestion buffer config.
+        #[serde(default)]
+        pub buffer_config: BufferConfig,
+
         /// The scan interval in seconds. Defaults to 30 seconds.
         #[serde(default = "default_scanning_interval_sec")]
         pub scanning_interval_sec: u32,

components/job-orchestration/job_orchestration/garbage_collector/archive_garbage_collector.py

@@ -199,7 +199,7 @@ async def archive_garbage_collector(clp_config: ClpConfig) -> None:
     validate_storage_type(archive_output_config, storage_engine)
 
     sweep_interval_secs = clp_config.garbage_collector.sweep_interval.archive * MIN_TO_SECONDS
-    recovery_file = clp_config.logs_directory / f"{ARCHIVE_GARBAGE_COLLECTOR_NAME}.tmp"
+    recovery_file = clp_config.tmp_directory / f"{ARCHIVE_GARBAGE_COLLECTOR_NAME}.tmp"
 
     logger.info(f"{ARCHIVE_GARBAGE_COLLECTOR_NAME} started.")
     try:

components/log-ingestor/src/ingestion_job_manager/clp_ingestion.rs

@@ -422,6 +422,7 @@ impl ClpDbIngestionConnector {
             db_pool: self.db_pool.clone(),
         };
         let base_config = config.as_base_config();
+        let buffer_config = config.as_buffer_config();
 
         let submitter = CompressionJobSubmitter::new(
             compression_state.clone(),
@@ -431,9 +432,9 @@ impl ClpDbIngestionConnector {
         );
 
         let listener = Listener::spawn(
-            Buffer::new(submitter, base_config.buffer_config.flush_threshold_bytes),
-            Duration::from_secs(base_config.buffer_config.timeout_sec),
-            base_config.buffer_config.channel_capacity,
+            Buffer::new(submitter, buffer_config.flush_threshold_bytes),
+            Duration::from_secs(buffer_config.timeout_sec),
+            buffer_config.channel_capacity,
         )?;
         let ingestion_state = ClpIngestionState {
             job_id,

components/log-ingestor/tests/test_ingestion_job.rs

@@ -270,8 +270,8 @@ async fn test_sqs_listener() -> Result<()> {
                     dataset: None,
                     timestamp_key: None,
                     unstructured: false,
-                    buffer_config: BufferConfig::default(),
                 },
+                buffer_config: BufferConfig::default(),
             },
         )
         .await
@@ -312,8 +312,8 @@ async fn test_s3_scanner() -> Result<()> {
             dataset: None,
             timestamp_key: None,
             unstructured: false,
-            buffer_config: BufferConfig::default(),
         },
+        buffer_config: BufferConfig::default(),
         scanning_interval_sec: 1,
         start_after: None,
     };

components/webui/client/src/pages/LogViewerLoadingPage/QueryStatus.tsx

@@ -90,7 +90,9 @@ const QueryStatus = () => {
                 setQueryState(QUERY_LOADING_STATE.LOADING);
 
                 const innerLogEventNum = parseResult.logEventIdx - data.begin_msg_ix + 1;
-                window.location.href = `/log-viewer/index.html?filePath=${data.path}` +
+                const filePath = encodeURIComponent(data.path);
+                window.location.href =
+                    `/log-viewer/index.html?filePath=${filePath}` +
                     `#logEventNum=${innerLogEventNum}`;
             })
             .catch((e: unknown) => {

components/webui/server/.env

@@ -18,3 +18,7 @@ PRESTO_SCHEMA=default
 # Security
 RATE_LIMIT=1000
 
+# S3
+CLP_STREAM_OUTPUT_AWS_ACCESS_KEY_ID=
+CLP_STREAM_OUTPUT_AWS_SECRET_ACCESS_KEY=
+