Update buckets (#672)

* updating to use new bucket in jvector account
* adding credentials step to workflow

Author: MarkWolters

.github/workflows/run-bench.yml

@@ -72,6 +72,9 @@ jobs:
     strategy:
       matrix: ${{ fromJSON(needs.generate-matrix.outputs.matrix) }}
     runs-on: ${{ matrix.isa }}
+    permissions:
+      id-token: write   # required for OIDC token request
+      contents: read
     steps:
       - name: verify-avx512
         run: |
@@ -140,6 +143,12 @@ jobs:
           mkdir -p jvector-examples/yaml-configs/dataset-catalogs
           echo "${{ secrets.PROTECTED_CATALOG_YAML }}" | base64 -d > jvector-examples/yaml-configs/dataset-catalogs/protected-catalog.yaml
 
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
+          aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
+
       # Create a directory to store benchmark results
       - name: Create results directory
         run: mkdir -p benchmark_results

jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/datasets/DataSetLoaderSimpleMFD.java

@@ -19,7 +19,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.yaml.snakeyaml.Yaml;
-import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
 import software.amazon.awssdk.regions.Region;
 import software.amazon.awssdk.services.s3.S3AsyncClient;
 import software.amazon.awssdk.transfer.s3.S3TransferManager;
@@ -188,6 +188,16 @@ static String redact(Object value) {
         String s = value.toString();
         if (s.isEmpty()) return s;
 
+        // Unescape URL-encoded path separators (%2F, %5C) before locating
+        // the leaf filename, so S3 keys with encoded slashes are handled correctly.
+        s = s.replace("%2F", "/").replace("%2f", "/")
+             .replace("%5C", "\\").replace("%5c", "\\");
+
+        // Find the last path delimiter using lastIndexOf.
+        // Both '/' and '\\' must be checked explicitly: path strings arrive from S3 URLs,
+        // Linux paths, and Windows paths, so the OS-local '\\' is not enough.
+        int lastDelim = Math.max(s.lastIndexOf('/'), s.lastIndexOf('\\'));
+
         var sb = new StringBuilder(s.length());
         int i = 0;
         while (i < s.length()) {
@@ -197,7 +207,8 @@ static String redact(Object value) {
                 i++;
             }
             String segment = s.substring(segStart, i);
-            sb.append(looksLikeSecret(segment) ? "[[redacted]]" : segment);
+            boolean isLeaf = segStart > lastDelim;
+            sb.append(!isLeaf && looksLikeSecret(segment) ? "[[redacted]]" : segment);
 
             // append the delimiter(s)
             while (i < s.length() && (s.charAt(i) == '/' || s.charAt(i) == '\\')) {
@@ -891,7 +902,7 @@ private void downloadFileS3(String s3Url, Path localPath) throws IOException {
     private static S3AsyncClient s3AsyncClient() {
         return S3AsyncClient.crtBuilder()
                 .region(Region.US_EAST_1)
-                .credentialsProvider(AnonymousCredentialsProvider.create())
+                .credentialsProvider(DefaultCredentialsProvider.create())
                 .targetThroughputInGbps(10.0)
                 .minimumPartSizeInBytes(8L * 1024 * 1024)
                 .build();

jvector-examples/src/test/java/io/github/jbellis/jvector/example/benchmarks/datasets/DataSetLoaderSimpleMFDTest.java

@@ -1217,7 +1217,7 @@ public void redactHandles0xPrefix() {
 
     @Test
     public void redactPreservesNonHashPaths() {
-        String url = "s3://jvector-datasets-public/datasets-clean/ada_002_100k_base.fvecs";
+        String url = "s3://vector-datasets-20260601/public/datasets-clean/ada_002_100k_base_99287.fvecs";
         assertEquals(url, DataSetLoaderSimpleMFD.redact(url), "No hash segments — should be unchanged");
     }
 

jvector-examples/yaml-configs/dataset-catalogs/datasets.md

@@ -26,7 +26,7 @@ Public datasets work out of the box. `public-catalog.yaml` uses
 
 ```sh
 # see what's available
-curl -L https://jvector-datasets-public.s3.us-east-1.amazonaws.com/datasets-clean/catalog_entries.yaml
+https://vector-datasets-20260601.s3.us-east-1.amazonaws.com/public/datasets-clean/catalog_entries.yaml
 ```
 
 Downloaded files are cached locally in `dataset_cache/public/` by default.

jvector-examples/yaml-configs/dataset-catalogs/public-catalog.yaml

@@ -1,10 +1,10 @@
 # Please do not modify this file.
 #
-# Hint: you can see what's available in our public-catalog by running
-# curl -L https://jvector-datasets-public.s3.us-east-1.amazonaws.com/datasets-clean/catalog_entries.yaml
+# Hint: you can see what's available in our public-catalog by running# curl -L https://jvector-datasets-public.s3.us-east-1.amazonaws.com/datasets-clean/catalog_entries.yaml
+# curl -L https://vector-datasets-20260601.s3.us-east-1.amazonaws.com/public/datasets-clean/catalog_entries.yaml
 
 _include:
-  url: s3://jvector-datasets-public/datasets-clean/catalog_entries.yaml
+  url: s3://vector-datasets-20260601/public/datasets-clean/catalog_entries.yaml
 
 # This sets the local cache directory for the datasets.
 # If it is not set, the dataset will be cached under dataset_cache/