CASSANALYTICS-155: Add IAM credential support for S3 storage transport

Adds IAM-based authentication as an alternative to static credentials
for S3 storage transport.

Also fixes test report collection in CI: corrects Gradle output paths
for both GitHub Actions (adds missing artifact uploads with if: always())
and CircleCI (fixes SRC_REPORT_DIR and store_test_results paths so real
JUnit XML is collected instead of synthesized fallbacks).

Patch by Jon Haddad; reviewed by Yifan Cai for CASSANALYTICS-155

Author: rustyrazorblade

.circleci/config.yml

@@ -116,7 +116,8 @@ commands:
             # files on each run, so without aggregation only the last class's XML
             # would survive, hiding most results from the CircleCI dashboard.
             # Drain reports into a per-class subdirectory after each iteration.
-            SRC_REPORT_DIR="$(pwd)/build/test-reports/integration"
+            SRC_REPORT_DIR="$(pwd)/build/test-results/cassandra-analytics-integration-tests/test"
+            HTML_REPORT_DIR="$(pwd)/build/reports/tests/cassandra-analytics-integration-tests/test"
             AGG_ROOT="$(pwd)/build/aggregated-test-reports/integration"
             mkdir -p "$AGG_ROOT"
             # collect up exit statuses for all of the test classes and exit with that result at the end.
@@ -133,10 +134,10 @@ commands:
                         mv "$f" "$DEST"/
                         MOVED=1
                     done
-                    for f in "$SRC_REPORT_DIR"/*.html; do
-                        [ -e "$f" ] || continue
-                        mv "$f" "$DEST"/ 2>/dev/null || true
-                    done
+                fi
+                if [ -d "$HTML_REPORT_DIR" ]; then
+                    mkdir -p "$DEST/html"
+                    cp -r "$HTML_REPORT_DIR/." "$DEST/html/"
                 fi
                 # If Gradle produced no XML (e.g. class-level crash before any
                 # @Test ran), synthesize a minimal JUnit record so CircleCI's
@@ -244,8 +245,8 @@ jobs:
 
       - store_artifacts:
           when: always
-          path: build/test-reports
-          destination: test-reports
+          path: build/test-results
+          destination: test-results
 
       - store_artifacts:
           when: always
@@ -254,7 +255,7 @@ jobs:
 
       - store_test_results:
           when: always
-          path: build/test-reports
+          path: build/test-results
 
   # Single parameterized integration-test job, invoked once per Cassandra/Scala
   # compatibility group from the workflow's `matrix:` block.
@@ -286,8 +287,8 @@ jobs:
 
       - store_artifacts:
           when: always
-          path: build/test-reports
-          destination: test-reports
+          path: build/aggregated-test-reports
+          destination: test-results
 
       - store_artifacts:
           when: always

.github/workflows/test.yaml

@@ -200,6 +200,15 @@ jobs:
           fi
 
           ./gradlew --stacktrace clean assemble check -x cassandra-analytics-integration-tests:test -Dcassandra.analytics.bridges.sstable_format=${{ matrix.sstable-format }}
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: unit-test-results-s${{ matrix.scala }}-${{ matrix.sstable-format }}-C${{ matrix.cassandra }}-Spark${{ matrix.spark }}-JDK${{ matrix.jdk }}
+          path: |
+            build/test-results/**
+            build/reports/tests/**
+          retention-days: 30
 
   integration-test:
     name: Integration test - ${{ matrix.config }} (${{ matrix.job_index }})
@@ -302,23 +311,23 @@ jobs:
             done
 
             test_id=$(date +%H%M%S)
-            mkdir -p test-reports/$test_id
+            mkdir -p "test-reports/$test_id/html"
 
             if [ $SKIP == "false" ]; then
               echo Executing test $TEST_NAME
               ./gradlew --stacktrace cassandra-analytics-integration-tests:test --tests $TEST_NAME --no-daemon || EXIT_STATUS=$?;
-              mv build/test-reports/* test-reports/$test_id
+              find build/test-results -name "*.xml" -exec cp {} "test-reports/$test_id/" \; 2>/dev/null || true
+              [ -d "build/reports/tests" ] && cp -r build/reports/tests/. "test-reports/$test_id/html/" 2>/dev/null || true
             else
               echo "Skipping test $TEST_NAME"
             fi
           done;
 
-          tar czf integration-tests.tar.gz test-reports/*
-
           exit $EXIT_STATUS
-      - name: Publish test results
+      - name: Upload test results
+        if: always()
         uses: actions/upload-artifact@v4
-        if: (!cancelled())
         with:
           name: integration-tests-${{ matrix.config }}-${{ matrix.job_index }}
-          path: integration-tests.tar.gz
+          path: test-reports/
+          retention-days: 30

CHANGES.txt

@@ -1,6 +1,7 @@
 0.5.0
 -----
  * Spark 4.0 Support (CASSANALYTICS-34)
+ * Add IAM credential support for S3 storage transport (CASSANALYTICS-155)
  * Make BulkWriterConfig extensible (CASSANALYTICS-168)
 
 0.4.0

analytics-sidecar-client-common/build.gradle

@@ -34,14 +34,6 @@ if (propertyWithDefault("artifactType", null) == "common") {
 test {
     useJUnitPlatform()
     maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1
-    reports {
-        junitXml.setRequired(true)
-        def destDir = Paths.get(rootProject.rootDir.absolutePath, "build", "test-results", "client-common").toFile()
-        println("Destination directory for client-common tests: ${destDir}")
-        junitXml.getOutputLocation().set(destDir)
-        html.setRequired(true)
-        html.getOutputLocation().set(destDir)
-    }
 }
 
 dependencies {

analytics-sidecar-client-common/src/main/java/org/apache/cassandra/sidecar/common/data/CredentialType.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sidecar.common.data;
+
+/**
+ * Declares the credential type used by a restore job to authenticate with cloud storage.
+ */
+public enum CredentialType
+{
+    /** Static STS credentials: accessKeyId, secretAccessKey, and sessionToken must all be present. */
+    STATIC,
+    /** IAM instance profile (or ECS task role / IRSA): only region is required; key fields must be absent. */
+    IAM
+}

analytics-sidecar-client-common/src/main/java/org/apache/cassandra/sidecar/common/data/RestoreJobConstants.java

@@ -58,6 +58,7 @@ public class RestoreJobConstants
     public static final String SLICE_COMPRESSED_SIZE = "sliceCompressedSize";
     public static final String SECRET_READ_CREDENTIALS = "readCredentials";
     public static final String SECRET_WRITE_CREDENTIALS = "writeCredentials";
+    public static final String JOB_CREDENTIAL_TYPE = "credentialType";
     public static final String CREDENTIALS_ACCESS_KEY_ID = "accessKeyId";
     public static final String CREDENTIALS_SECRET_ACCESS_KEY = "secretAccessKey";
     public static final String CREDENTIALS_SESSION_TOKEN = "sessionToken";

analytics-sidecar-client-common/src/main/java/org/apache/cassandra/sidecar/common/data/RestoreJobSecrets.java

@@ -31,6 +31,21 @@
  */
 public class RestoreJobSecrets
 {
+    /**
+     * Creates a {@link RestoreJobSecrets} for IAM instance profile mode.
+     * Only the regions are provided; the sidecar resolves credentials via the AWS default credential chain.
+     *
+     * @param readRegion  the AWS region of the S3 bucket used for reading
+     * @param writeRegion the AWS region of the S3 bucket used for writing
+     * @return region-only {@link RestoreJobSecrets} signalling IAM mode
+     */
+    public static RestoreJobSecrets iamMode(String readRegion, String writeRegion)
+    {
+        StorageCredentials read = StorageCredentials.builder().region(readRegion).build();
+        StorageCredentials write = StorageCredentials.builder().region(writeRegion).build();
+        return new RestoreJobSecrets(read, write);
+    }
+
     private final StorageCredentials writeCredentials;
     private final StorageCredentials readCredentials;
 

analytics-sidecar-client-common/src/main/java/org/apache/cassandra/sidecar/common/data/StorageCredentials.java

@@ -21,21 +21,26 @@
 import java.util.Objects;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import org.jetbrains.annotations.Nullable;
 
 import static org.apache.cassandra.sidecar.common.data.RestoreJobConstants.CREDENTIALS_ACCESS_KEY_ID;
 import static org.apache.cassandra.sidecar.common.data.RestoreJobConstants.CREDENTIALS_REGION;
 import static org.apache.cassandra.sidecar.common.data.RestoreJobConstants.CREDENTIALS_SECRET_ACCESS_KEY;
 import static org.apache.cassandra.sidecar.common.data.RestoreJobConstants.CREDENTIALS_SESSION_TOKEN;
 
 /**
- * Used for storing credential details needed for either reading/writing to S3
+ * Used for storing credential details needed for either reading/writing to S3.
+ * In IAM mode only {@code region} is required; the key fields are absent and the sidecar
+ * resolves credentials via the AWS default credential chain.
  */
+@JsonInclude(JsonInclude.Include.NON_NULL)
 public class StorageCredentials
 {
-    private final String accessKeyId;
-    private final String secretAccessKey;
-    private final String sessionToken;
+    private final @Nullable String accessKeyId;
+    private final @Nullable String secretAccessKey;
+    private final @Nullable String sessionToken;
     private final String region;
 
     public static Builder builder()
@@ -44,14 +49,11 @@ public static Builder builder()
     }
 
     @JsonCreator
-    public StorageCredentials(@JsonProperty(CREDENTIALS_ACCESS_KEY_ID) String accessKeyId,
-                              @JsonProperty(CREDENTIALS_SECRET_ACCESS_KEY) String secretAccessKey,
-                              @JsonProperty(CREDENTIALS_SESSION_TOKEN) String sessionToken,
+    public StorageCredentials(@JsonProperty(CREDENTIALS_ACCESS_KEY_ID) @Nullable String accessKeyId,
+                              @JsonProperty(CREDENTIALS_SECRET_ACCESS_KEY) @Nullable String secretAccessKey,
+                              @JsonProperty(CREDENTIALS_SESSION_TOKEN) @Nullable String sessionToken,
                               @JsonProperty(CREDENTIALS_REGION) String region)
     {
-        Objects.requireNonNull(accessKeyId, "accessKeyId must be supplied");
-        Objects.requireNonNull(secretAccessKey, "secretAccessKey must be supplied");
-        Objects.requireNonNull(sessionToken, "sessionToken must be supplied");
         Objects.requireNonNull(region, "region must be supplied");
         this.accessKeyId = accessKeyId;
         this.secretAccessKey = secretAccessKey;
@@ -60,19 +62,19 @@ public StorageCredentials(@JsonProperty(CREDENTIALS_ACCESS_KEY_ID) String access
     }
 
     @JsonProperty(CREDENTIALS_ACCESS_KEY_ID)
-    public String accessKeyId()
+    public @Nullable String accessKeyId()
     {
         return accessKeyId;
     }
 
     @JsonProperty(CREDENTIALS_SECRET_ACCESS_KEY)
-    public String secretAccessKey()
+    public @Nullable String secretAccessKey()
     {
         return secretAccessKey;
     }
 
     @JsonProperty(CREDENTIALS_SESSION_TOKEN)
-    public String sessionToken()
+    public @Nullable String sessionToken()
     {
         return sessionToken;
     }
@@ -83,6 +85,15 @@ public String region()
         return region;
     }
 
+    /**
+     * Returns true if all three key fields are non-null, indicating static AWS credentials are present.
+     * A false result means key fields are absent, which is expected in IAM mode.
+     */
+    public boolean hasStaticCredentials()
+    {
+        return accessKeyId != null && secretAccessKey != null && sessionToken != null;
+    }
+
     /**
      * @return secrets string with redacted values
      */
@@ -123,13 +134,13 @@ public int hashCode()
     }
 
     /**
-     * Builds the RestoreJobSecrets
+     * Builds the StorageCredentials
      */
     public static class Builder
     {
-        private String accessKeyId;
-        private String secretAccessKey;
-        private String sessionToken;
+        private @Nullable String accessKeyId;
+        private @Nullable String secretAccessKey;
+        private @Nullable String sessionToken;
         private String region;
 
         private Builder()