Merge pull request #80 from konstibob/fix/sub_shard_read

Fix S3 suffix byte-range requests when reading sharded chunks

Author: normanrz

src/main/java/dev/zarr/zarrjava/store/S3Store.java

@@ -1,18 +1,29 @@
 package dev.zarr.zarrjava.store;
 
-import dev.zarr.zarrjava.utils.Utils;
-import software.amazon.awssdk.core.ResponseInputStream;
-import software.amazon.awssdk.core.sync.RequestBody;
-import software.amazon.awssdk.services.s3.S3Client;
-import software.amazon.awssdk.services.s3.model.*;
-
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.ByteBuffer;
 import java.util.stream.Stream;
 
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import dev.zarr.zarrjava.utils.Utils;
+import software.amazon.awssdk.core.ResponseInputStream;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.CommonPrefix;
+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectResponse;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
+import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.S3Exception;
+import software.amazon.awssdk.services.s3.model.S3Object;
+
 public class S3Store implements Store, Store.ListableStore {
 
     @Nonnull
@@ -82,7 +93,9 @@ public ByteBuffer get(String[] keys, long start) {
         GetObjectRequest req = GetObjectRequest.builder()
                 .bucket(bucketName)
                 .key(resolveKeys(keys))
-                .range(String.format("bytes=%d-", start))
+                  .range(start < 0 // negative start implies indexing from the end, i.e. last bytes
+                    ? String.format("bytes=%d", start)
+                    : String.format("bytes=%d-", start))
                 .build();
         return get(req);
     }

src/test/java/dev/zarr/zarrjava/store/OnlineS3StoreTest.java

@@ -60,6 +60,24 @@ public void testGet() {
         Assertions.assertEquals(10, bufferWithStartAndEnd.remaining());
     }
 
+    @Test
+    public void testReadSuffix() {
+        StoreHandle storeHandle = storeHandleWithData();
+        ByteBuffer fullBuffer = storeHandle.read();
+        long size = fullBuffer.remaining();
+        Assertions.assertTrue(size >= 20, "Store size is too small to test suffix read");
+
+        ByteBuffer suffixBuffer = storeHandle.read(-10);
+        Assertions.assertEquals(10, suffixBuffer.remaining());
+
+        byte[] expectedBytes = new byte[10];
+        fullBuffer.position((int) (size - 10));
+        fullBuffer.get(expectedBytes);
+        byte[] actualBytes = new byte[10];
+        suffixBuffer.get(actualBytes);
+        Assertions.assertArrayEquals(expectedBytes, actualBytes);
+    }
+
     @Override
     StoreHandle storeHandleWithData() {
         return storeHandle.resolve("zarr.json");