Skip to content

Commit 2927ca7

Browse files
brokkoli71brokkoli71
committed
Merge branch 'main' into zip-store
# Conflicts: # src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java # src/main/java/dev/zarr/zarrjava/store/HttpStore.java # src/main/java/dev/zarr/zarrjava/store/MemoryStore.java # src/main/java/dev/zarr/zarrjava/store/S3Store.java # src/main/java/dev/zarr/zarrjava/store/Store.java # src/main/java/dev/zarr/zarrjava/store/StoreHandle.java # src/main/java/dev/zarr/zarrjava/v2/Group.java # src/main/java/dev/zarr/zarrjava/v3/Group.java # src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java # src/test/java/dev/zarr/zarrjava/ZarrV2Test.java
2 parents e8fb440 + 93b10bb commit 2927ca7

77 files changed

Lines changed: 4704 additions & 4657 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.gitignore‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,9 @@ target/
44
!**/src/test/**/target/
55

66
### IntelliJ IDEA ###
7-
.idea/
7+
.idea/*
8+
!.idea/codeStyles/
9+
!.idea/codeStyles/**
810
*.iws
911
*.iml
1012
*.ipr
@@ -43,4 +45,4 @@ build/
4345
/main.py
4446
/pyproject.toml
4547
/uv.lock
46-
**/__pycache__
48+
**/__pycache__

‎.idea/codeStyles/codeStyleConfig.xml‎

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎README.md‎

Lines changed: 31 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
This repository contains a Java implementation of Zarr version 2 and 3.
44

55
## Usage
6+
67
```java
78
import dev.zarr.zarrjava.store.FilesystemStore;
89
import dev.zarr.zarrjava.store.HttpStore;
@@ -11,40 +12,54 @@ import dev.zarr.zarrjava.v3.DataType;
1112
import dev.zarr.zarrjava.v3.Group;
1213

1314
Group hierarchy = Group.open(
14-
new HttpStore("https://static.webknossos.org/data/zarr_v3")
15-
.resolve("l4_sample")
15+
new HttpStore("https://static.webknossos.org/data/zarr_v3")
16+
.resolve("l4_sample")
1617
);
1718
Group color = (Group) hierarchy.get("color");
1819
Array array = (Array) color.get("1");
1920
ucar.ma2.Array outArray = array.read(
20-
new long[]{0, 3073, 3073, 513}, // offset
21-
new int[]{1, 64, 64, 64} // shape
21+
new long[]{0, 3073, 3073, 513}, // offset
22+
new int[]{1, 64, 64, 64} // shape
2223
);
2324

2425
Array array = Array.create(
25-
new FilesystemStore("/path/to/zarr").resolve("array"),
26-
Array.metadataBuilder()
27-
.withShape(1, 4096, 4096, 1536)
28-
.withDataType(DataType.UINT32)
29-
.withChunkShape(1, 1024, 1024, 1024)
30-
.withFillValue(0)
31-
.withCodecs(c -> c.withSharding(new int[]{1, 32, 32, 32}, c1 -> c1.withBlosc()))
32-
.build()
26+
new FilesystemStore("/path/to/zarr").resolve("array"),
27+
Array.metadataBuilder()
28+
.withShape(1, 4096, 4096, 1536)
29+
.withDataType(DataType.UINT32)
30+
.withChunkShape(1, 1024, 1024, 1024)
31+
.withFillValue(0)
32+
.withCodecs(c -> c.withSharding(new int[]{1, 32, 32, 32}, c1 -> c1.withBlosc()))
33+
.build()
3334
);
3435
ucar.ma2.Array data = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1, 1024, 1024, 1024});
35-
array.write(
36-
new long[]{0, 0, 0, 0}, // offset
37-
data
36+
array.
37+
38+
write(
39+
new long[] {
40+
0, 0, 0, 0
41+
}, // offset
42+
data
3843
);
3944
```
45+
4046
## Development Start-Guide
4147

4248
### Run Tests Locally
43-
To be able to run the tests locally, make sure to have `python3.11` and `uv` installed.
49+
50+
To be able to run the tests locally, make sure to have `python3.11` and `uv` installed.
4451

4552
Furthermore, you will need the `l4_sample` test data:
4653

4754
`curl https://static.webknossos.org/data/zarr_v3/l4_sample.zip -o testdata/l4_sample.zip
4855
&& cd testdata
4956
&& unzip l4_sample.zip
5057
`
58+
59+
### Code Style & Formatting
60+
61+
This project uses IntelliJ IDEA default Java formatting
62+
63+
Before submitting changes, please run:
64+
65+
- IntelliJ: `Reformat Code` and `Optimize Imports`

‎pom.xml‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -199,10 +199,10 @@
199199
<version>3.5.0</version>
200200
<executions>
201201
<execution>
202-
<id>attach-javadoc</id>
203-
<goals>
204-
<goal>jar</goal>
205-
</goals>
202+
<id>attach-javadoc</id>
203+
<goals>
204+
<goal>jar</goal>
205+
</goals>
206206
</execution>
207207
</executions>
208208
<configuration>
@@ -215,14 +215,14 @@
215215
<version>1.6</version>
216216
<executions>
217217
<execution>
218-
<id>sign-artifacts</id>
219-
<phase>verify</phase>
220-
<goals>
221-
<goal>sign</goal>
222-
</goals>
223-
<configuration>
224-
<keyname>9F88D86AD9A0D91E</keyname>
225-
</configuration>
218+
<id>sign-artifacts</id>
219+
<phase>verify</phase>
220+
<goals>
221+
<goal>sign</goal>
222+
</goals>
223+
<configuration>
224+
<keyname>9F88D86AD9A0D91E</keyname>
225+
</configuration>
226226
</execution>
227227
</executions>
228228
</plugin>

‎src/main/java/dev/zarr/zarrjava/ZarrException.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@
22

33
public class ZarrException extends Exception {
44

5-
public ZarrException(String message, Throwable cause) {
6-
super(message, cause);
7-
}
5+
public ZarrException(String message, Throwable cause) {
6+
super(message, cause);
7+
}
88

9-
public ZarrException(String message) {
10-
super(message);
11-
}
9+
public ZarrException(String message) {
10+
super(message);
11+
}
1212
}

‎src/main/java/dev/zarr/zarrjava/core/Array.java‎

Lines changed: 66 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
package dev.zarr.zarrjava.core;
22

33
import dev.zarr.zarrjava.ZarrException;
4+
import dev.zarr.zarrjava.core.codec.CodecPipeline;
45
import dev.zarr.zarrjava.store.FilesystemStore;
56
import dev.zarr.zarrjava.store.StoreHandle;
67
import dev.zarr.zarrjava.utils.IndexingUtils;
78
import dev.zarr.zarrjava.utils.MultiArrayUtils;
89
import dev.zarr.zarrjava.utils.Utils;
9-
import dev.zarr.zarrjava.core.codec.CodecPipeline;
1010
import ucar.ma2.InvalidRangeException;
1111

1212
import javax.annotation.Nonnull;
@@ -21,7 +21,6 @@
2121
public abstract class Array extends AbstractNode {
2222

2323
protected CodecPipeline codecPipeline;
24-
public abstract ArrayMetadata metadata();
2524

2625
protected Array(StoreHandle storeHandle) throws ZarrException {
2726
super(storeHandle);
@@ -47,7 +46,8 @@ public static Array open(StoreHandle storeHandle) throws IOException, ZarrExcept
4746
throw new ZarrException("No Zarr array found at the specified location.");
4847
}
4948
}
50-
/**
49+
50+
/**
5151
* Opens an existing Zarr array at a specified storage location. Automatically detects the Zarr version.
5252
*
5353
* @param path the storage location of the Zarr array
@@ -69,6 +69,8 @@ public static Array open(String path) throws IOException, ZarrException {
6969
return open(Paths.get(path));
7070
}
7171

72+
public abstract ArrayMetadata metadata();
73+
7274
/**
7375
* Writes a ucar.ma2.Array into the Zarr array at a specified offset. The shape of the Zarr array
7476
* needs be large enough for the write.
@@ -94,32 +96,32 @@ public void write(long[] offset, ucar.ma2.Array array, boolean parallel) {
9496
chunkStream = chunkStream.parallel();
9597
}
9698
chunkStream.forEach(
97-
chunkCoords -> {
98-
try {
99-
final IndexingUtils.ChunkProjection chunkProjection =
100-
IndexingUtils.computeProjection(chunkCoords, metadata.shape, chunkShape, offset,
101-
shape
102-
);
103-
104-
ucar.ma2.Array chunkArray;
105-
if (IndexingUtils.isFullChunk(chunkProjection.chunkOffset, chunkProjection.shape,
106-
chunkShape
107-
)) {
108-
chunkArray = array.sectionNoReduce(chunkProjection.outOffset,
109-
chunkProjection.shape,
110-
null
111-
);
112-
} else {
113-
chunkArray = readChunk(chunkCoords);
114-
MultiArrayUtils.copyRegion(array, chunkProjection.outOffset, chunkArray,
115-
chunkProjection.chunkOffset, chunkProjection.shape
116-
);
99+
chunkCoords -> {
100+
try {
101+
final IndexingUtils.ChunkProjection chunkProjection =
102+
IndexingUtils.computeProjection(chunkCoords, metadata.shape, chunkShape, offset,
103+
shape
104+
);
105+
106+
ucar.ma2.Array chunkArray;
107+
if (IndexingUtils.isFullChunk(chunkProjection.chunkOffset, chunkProjection.shape,
108+
chunkShape
109+
)) {
110+
chunkArray = array.sectionNoReduce(chunkProjection.outOffset,
111+
chunkProjection.shape,
112+
null
113+
);
114+
} else {
115+
chunkArray = readChunk(chunkCoords);
116+
MultiArrayUtils.copyRegion(array, chunkProjection.outOffset, chunkArray,
117+
chunkProjection.chunkOffset, chunkProjection.shape
118+
);
119+
}
120+
writeChunk(chunkCoords, chunkArray);
121+
} catch (ZarrException | InvalidRangeException e) {
122+
throw new RuntimeException(e);
117123
}
118-
writeChunk(chunkCoords, chunkArray);
119-
} catch (ZarrException | InvalidRangeException e) {
120-
throw new RuntimeException(e);
121-
}
122-
});
124+
});
123125

124126
}
125127

@@ -246,7 +248,7 @@ boolean chunkIsInArray(long[] chunkCoords) {
246248
final int[] chunkShape = metadata().chunkShape();
247249
for (int dimIdx = 0; dimIdx < metadata().ndim(); dimIdx++) {
248250
if (chunkCoords[dimIdx] < 0
249-
|| chunkCoords[dimIdx] * chunkShape[dimIdx] >= metadata().shape[dimIdx]) {
251+
|| chunkCoords[dimIdx] * chunkShape[dimIdx] >= metadata().shape[dimIdx]) {
250252
return false;
251253
}
252254
}
@@ -282,47 +284,47 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape, final boolean
282284
}
283285

284286
final ucar.ma2.Array outputArray = ucar.ma2.Array.factory(metadata.dataType().getMA2DataType(),
285-
shape);
287+
shape);
286288
Stream<long[]> chunkStream = Arrays.stream(IndexingUtils.computeChunkCoords(metadata.shape, chunkShape, offset, shape));
287289
if (parallel) {
288290
chunkStream = chunkStream.parallel();
289291
}
290292
chunkStream.forEach(
291-
chunkCoords -> {
292-
try {
293-
final IndexingUtils.ChunkProjection chunkProjection =
294-
IndexingUtils.computeProjection(chunkCoords, metadata.shape, chunkShape, offset,
295-
shape
296-
);
297-
298-
if (chunkIsInArray(chunkCoords)) {
299-
MultiArrayUtils.copyRegion(metadata.allocateFillValueChunk(),
300-
chunkProjection.chunkOffset, outputArray, chunkProjection.outOffset,
301-
chunkProjection.shape
302-
);
303-
}
304-
305-
final String[] chunkKeys = metadata.chunkKeyEncoding().encodeChunkKey(chunkCoords);
306-
final StoreHandle chunkHandle = storeHandle.resolve(chunkKeys);
307-
if (!chunkHandle.exists()) {
308-
return;
293+
chunkCoords -> {
294+
try {
295+
final IndexingUtils.ChunkProjection chunkProjection =
296+
IndexingUtils.computeProjection(chunkCoords, metadata.shape, chunkShape, offset,
297+
shape
298+
);
299+
300+
if (chunkIsInArray(chunkCoords)) {
301+
MultiArrayUtils.copyRegion(metadata.allocateFillValueChunk(),
302+
chunkProjection.chunkOffset, outputArray, chunkProjection.outOffset,
303+
chunkProjection.shape
304+
);
305+
}
306+
307+
final String[] chunkKeys = metadata.chunkKeyEncoding().encodeChunkKey(chunkCoords);
308+
final StoreHandle chunkHandle = storeHandle.resolve(chunkKeys);
309+
if (!chunkHandle.exists()) {
310+
return;
311+
}
312+
if (codecPipeline.supportsPartialDecode()) {
313+
final ucar.ma2.Array chunkArray = codecPipeline.decodePartial(chunkHandle,
314+
Utils.toLongArray(chunkProjection.chunkOffset), chunkProjection.shape);
315+
MultiArrayUtils.copyRegion(chunkArray, new int[metadata.ndim()], outputArray,
316+
chunkProjection.outOffset, chunkProjection.shape
317+
);
318+
} else {
319+
MultiArrayUtils.copyRegion(readChunk(chunkCoords), chunkProjection.chunkOffset,
320+
outputArray, chunkProjection.outOffset, chunkProjection.shape
321+
);
322+
}
323+
324+
} catch (ZarrException e) {
325+
throw new RuntimeException(e);
309326
}
310-
if (codecPipeline.supportsPartialDecode()) {
311-
final ucar.ma2.Array chunkArray = codecPipeline.decodePartial(chunkHandle,
312-
Utils.toLongArray(chunkProjection.chunkOffset), chunkProjection.shape);
313-
MultiArrayUtils.copyRegion(chunkArray, new int[metadata.ndim()], outputArray,
314-
chunkProjection.outOffset, chunkProjection.shape
315-
);
316-
} else {
317-
MultiArrayUtils.copyRegion(readChunk(chunkCoords), chunkProjection.chunkOffset,
318-
outputArray, chunkProjection.outOffset, chunkProjection.shape
319-
);
320-
}
321-
322-
} catch (ZarrException e) {
323-
throw new RuntimeException(e);
324-
}
325-
});
327+
});
326328
return outputArray;
327329
}
328330

0 commit comments

Comments
 (0)