Skip to content
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ target/
!**/src/test/**/target/

### IntelliJ IDEA ###
.idea/
.idea/*
!.idea/codeStyles/
!.idea/codeStyles/**
*.iws
*.iml
*.ipr
Expand Down Expand Up @@ -43,4 +45,4 @@ build/
/main.py
/pyproject.toml
/uv.lock
**/__pycache__
**/__pycache__
5 changes: 5 additions & 0 deletions .idea/codeStyles/codeStyleConfig.xml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

47 changes: 31 additions & 16 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
This repository contains a Java implementation of Zarr version 2 and 3.

## Usage

```java
import dev.zarr.zarrjava.store.FilesystemStore;
import dev.zarr.zarrjava.store.HttpStore;
Expand All @@ -11,40 +12,54 @@ import dev.zarr.zarrjava.v3.DataType;
import dev.zarr.zarrjava.v3.Group;

Group hierarchy = Group.open(
new HttpStore("https://static.webknossos.org/data/zarr_v3")
.resolve("l4_sample")
new HttpStore("https://static.webknossos.org/data/zarr_v3")
.resolve("l4_sample")
);
Group color = (Group) hierarchy.get("color");
Array array = (Array) color.get("1");
ucar.ma2.Array outArray = array.read(
new long[]{0, 3073, 3073, 513}, // offset
new int[]{1, 64, 64, 64} // shape
new long[]{0, 3073, 3073, 513}, // offset
new int[]{1, 64, 64, 64} // shape
);

Array array = Array.create(
new FilesystemStore("/path/to/zarr").resolve("array"),
Array.metadataBuilder()
.withShape(1, 4096, 4096, 1536)
.withDataType(DataType.UINT32)
.withChunkShape(1, 1024, 1024, 1024)
.withFillValue(0)
.withCodecs(c -> c.withSharding(new int[]{1, 32, 32, 32}, c1 -> c1.withBlosc()))
.build()
new FilesystemStore("/path/to/zarr").resolve("array"),
Array.metadataBuilder()
.withShape(1, 4096, 4096, 1536)
.withDataType(DataType.UINT32)
.withChunkShape(1, 1024, 1024, 1024)
.withFillValue(0)
.withCodecs(c -> c.withSharding(new int[]{1, 32, 32, 32}, c1 -> c1.withBlosc()))
.build()
);
ucar.ma2.Array data = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1, 1024, 1024, 1024});
array.write(
new long[]{0, 0, 0, 0}, // offset
data
array.

write(
new long[] {
0, 0, 0, 0
}, // offset
data
);
```

## Development Start-Guide

### Run Tests Locally
To be able to run the tests locally, make sure to have `python3.11` and `uv` installed.

To be able to run the tests locally, make sure to have `python3.11` and `uv` installed.

Furthermore, you will need the `l4_sample` test data:

`curl https://static.webknossos.org/data/zarr_v3/l4_sample.zip -o testdata/l4_sample.zip
&& cd testdata
&& unzip l4_sample.zip
`

### Code Style & Formatting

This project uses IntelliJ IDEA default Java formatting

Before submitting changes, please run:

- IntelliJ: `Reformat Code` and `Optimize Imports`
24 changes: 12 additions & 12 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -194,10 +194,10 @@
<version>3.5.0</version>
<executions>
<execution>
<id>attach-javadoc</id>
<goals>
<goal>jar</goal>
</goals>
<id>attach-javadoc</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
<configuration>
Expand All @@ -210,14 +210,14 @@
<version>1.6</version>
<executions>
<execution>
<id>sign-artifacts</id>
<phase>verify</phase>
<goals>
<goal>sign</goal>
</goals>
<configuration>
<keyname>9F88D86AD9A0D91E</keyname>
</configuration>
<id>sign-artifacts</id>
<phase>verify</phase>
<goals>
<goal>sign</goal>
</goals>
<configuration>
<keyname>9F88D86AD9A0D91E</keyname>
</configuration>
</execution>
</executions>
</plugin>
Expand Down
12 changes: 6 additions & 6 deletions src/main/java/dev/zarr/zarrjava/ZarrException.java
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@

public class ZarrException extends Exception {

public ZarrException(String message, Throwable cause) {
super(message, cause);
}
public ZarrException(String message, Throwable cause) {
super(message, cause);
}

public ZarrException(String message) {
super(message);
}
public ZarrException(String message) {
super(message);
}
}
130 changes: 66 additions & 64 deletions src/main/java/dev/zarr/zarrjava/core/Array.java
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
package dev.zarr.zarrjava.core;

import dev.zarr.zarrjava.ZarrException;
import dev.zarr.zarrjava.core.codec.CodecPipeline;
import dev.zarr.zarrjava.store.FilesystemStore;
import dev.zarr.zarrjava.store.StoreHandle;
import dev.zarr.zarrjava.utils.IndexingUtils;
import dev.zarr.zarrjava.utils.MultiArrayUtils;
import dev.zarr.zarrjava.utils.Utils;
import dev.zarr.zarrjava.core.codec.CodecPipeline;
import ucar.ma2.InvalidRangeException;

import javax.annotation.Nonnull;
Expand All @@ -21,7 +21,6 @@
public abstract class Array extends AbstractNode {

protected CodecPipeline codecPipeline;
public abstract ArrayMetadata metadata();

protected Array(StoreHandle storeHandle) throws ZarrException {
super(storeHandle);
Expand All @@ -47,7 +46,8 @@ public static Array open(StoreHandle storeHandle) throws IOException, ZarrExcept
throw new ZarrException("No Zarr array found at the specified location.");
}
}
/**

/**
* Opens an existing Zarr array at a specified storage location. Automatically detects the Zarr version.
*
* @param path the storage location of the Zarr array
Expand All @@ -69,6 +69,8 @@ public static Array open(String path) throws IOException, ZarrException {
return open(Paths.get(path));
}

public abstract ArrayMetadata metadata();

/**
* Writes a ucar.ma2.Array into the Zarr array at a specified offset. The shape of the Zarr array
* needs be large enough for the write.
Expand All @@ -94,32 +96,32 @@ public void write(long[] offset, ucar.ma2.Array array, boolean parallel) {
chunkStream = chunkStream.parallel();
}
chunkStream.forEach(
chunkCoords -> {
try {
final IndexingUtils.ChunkProjection chunkProjection =
IndexingUtils.computeProjection(chunkCoords, metadata.shape, chunkShape, offset,
shape
);

ucar.ma2.Array chunkArray;
if (IndexingUtils.isFullChunk(chunkProjection.chunkOffset, chunkProjection.shape,
chunkShape
)) {
chunkArray = array.sectionNoReduce(chunkProjection.outOffset,
chunkProjection.shape,
null
);
} else {
chunkArray = readChunk(chunkCoords);
MultiArrayUtils.copyRegion(array, chunkProjection.outOffset, chunkArray,
chunkProjection.chunkOffset, chunkProjection.shape
);
chunkCoords -> {
try {
final IndexingUtils.ChunkProjection chunkProjection =
IndexingUtils.computeProjection(chunkCoords, metadata.shape, chunkShape, offset,
shape
);

ucar.ma2.Array chunkArray;
if (IndexingUtils.isFullChunk(chunkProjection.chunkOffset, chunkProjection.shape,
chunkShape
)) {
chunkArray = array.sectionNoReduce(chunkProjection.outOffset,
chunkProjection.shape,
null
);
} else {
chunkArray = readChunk(chunkCoords);
MultiArrayUtils.copyRegion(array, chunkProjection.outOffset, chunkArray,
chunkProjection.chunkOffset, chunkProjection.shape
);
}
writeChunk(chunkCoords, chunkArray);
} catch (ZarrException | InvalidRangeException e) {
throw new RuntimeException(e);
}
writeChunk(chunkCoords, chunkArray);
} catch (ZarrException | InvalidRangeException e) {
throw new RuntimeException(e);
}
});
});

}

Expand Down Expand Up @@ -246,7 +248,7 @@ boolean chunkIsInArray(long[] chunkCoords) {
final int[] chunkShape = metadata().chunkShape();
for (int dimIdx = 0; dimIdx < metadata().ndim(); dimIdx++) {
if (chunkCoords[dimIdx] < 0
|| chunkCoords[dimIdx] * chunkShape[dimIdx] >= metadata().shape[dimIdx]) {
|| chunkCoords[dimIdx] * chunkShape[dimIdx] >= metadata().shape[dimIdx]) {
return false;
}
}
Expand Down Expand Up @@ -282,47 +284,47 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape, final boolean
}

final ucar.ma2.Array outputArray = ucar.ma2.Array.factory(metadata.dataType().getMA2DataType(),
shape);
shape);
Stream<long[]> chunkStream = Arrays.stream(IndexingUtils.computeChunkCoords(metadata.shape, chunkShape, offset, shape));
if (parallel) {
chunkStream = chunkStream.parallel();
}
chunkStream.forEach(
chunkCoords -> {
try {
final IndexingUtils.ChunkProjection chunkProjection =
IndexingUtils.computeProjection(chunkCoords, metadata.shape, chunkShape, offset,
shape
);

if (chunkIsInArray(chunkCoords)) {
MultiArrayUtils.copyRegion(metadata.allocateFillValueChunk(),
chunkProjection.chunkOffset, outputArray, chunkProjection.outOffset,
chunkProjection.shape
);
}

final String[] chunkKeys = metadata.chunkKeyEncoding().encodeChunkKey(chunkCoords);
final StoreHandle chunkHandle = storeHandle.resolve(chunkKeys);
if (!chunkHandle.exists()) {
return;
chunkCoords -> {
try {
final IndexingUtils.ChunkProjection chunkProjection =
IndexingUtils.computeProjection(chunkCoords, metadata.shape, chunkShape, offset,
shape
);

if (chunkIsInArray(chunkCoords)) {
MultiArrayUtils.copyRegion(metadata.allocateFillValueChunk(),
chunkProjection.chunkOffset, outputArray, chunkProjection.outOffset,
chunkProjection.shape
);
}

final String[] chunkKeys = metadata.chunkKeyEncoding().encodeChunkKey(chunkCoords);
final StoreHandle chunkHandle = storeHandle.resolve(chunkKeys);
if (!chunkHandle.exists()) {
return;
}
if (codecPipeline.supportsPartialDecode()) {
final ucar.ma2.Array chunkArray = codecPipeline.decodePartial(chunkHandle,
Utils.toLongArray(chunkProjection.chunkOffset), chunkProjection.shape);
MultiArrayUtils.copyRegion(chunkArray, new int[metadata.ndim()], outputArray,
chunkProjection.outOffset, chunkProjection.shape
);
} else {
MultiArrayUtils.copyRegion(readChunk(chunkCoords), chunkProjection.chunkOffset,
outputArray, chunkProjection.outOffset, chunkProjection.shape
);
}

} catch (ZarrException e) {
throw new RuntimeException(e);
}
if (codecPipeline.supportsPartialDecode()) {
final ucar.ma2.Array chunkArray = codecPipeline.decodePartial(chunkHandle,
Utils.toLongArray(chunkProjection.chunkOffset), chunkProjection.shape);
MultiArrayUtils.copyRegion(chunkArray, new int[metadata.ndim()], outputArray,
chunkProjection.outOffset, chunkProjection.shape
);
} else {
MultiArrayUtils.copyRegion(readChunk(chunkCoords), chunkProjection.chunkOffset,
outputArray, chunkProjection.outOffset, chunkProjection.shape
);
}

} catch (ZarrException e) {
throw new RuntimeException(e);
}
});
});
return outputArray;
}

Expand Down
Loading
Loading