Skip to content

Commit 8fb9c4f

Browse files
TonioFTonioF
authored
Merge pull request #25 from bcdev/toniof-xxx-harmonize_with_zarr_devs
Harmonize jzarr
2 parents 533c8bb + 78ecf8f commit 8fb9c4f

19 files changed

Lines changed: 475 additions & 200 deletions

‎pom.xml‎

Lines changed: 17 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,20 @@
3232
</scm>
3333

3434
<developers>
35+
<developer>
36+
<id>SabineEmbacher</id>
37+
<name>Sabine Embacher</name>
38+
<roles>
39+
<role>maintainer</role>
40+
</roles>
41+
</developer>
42+
<developer>
43+
<id>TonioF</id>
44+
<name>Tonio Fincke</name>
45+
<roles>
46+
<role>maintainer</role>
47+
</roles>
48+
</developer>
3549
<developer>
3650
<id>joshmoore</id>
3751
<name>Josh Moore</name>
@@ -119,23 +133,7 @@
119133
<dependency>
120134
<groupId>org.assertj</groupId>
121135
<artifactId>assertj-core</artifactId>
122-
<version>3.19.0</version>
123-
<scope>test</scope>
124-
</dependency>
125-
<dependency>
126-
<groupId>org.lasersonlab</groupId>
127-
<artifactId>s3fs</artifactId>
128-
<version>2.2.3</version>
129-
<exclusions>
130-
<exclusion>
131-
<groupId>com.fasterxml.jackson.core</groupId>
132-
<artifactId>jackson-databind</artifactId>
133-
</exclusion>
134-
<exclusion>
135-
<groupId>com.fasterxml.jackson.dataformat</groupId>
136-
<artifactId>jackson-dataformat-cbor</artifactId>
137-
</exclusion>
138-
</exclusions>
136+
<version>3.27.7</version>
139137
<scope>test</scope>
140138
</dependency>
141139
</dependencies>
@@ -154,7 +152,7 @@
154152
<repositories>
155153
<repository>
156154
<id>unidata.releases</id>
157-
<url>https://artifacts.unidata.ucar.edu/content/repositories/unidata-releases</url>
155+
<url>https://artifacts.unidata.ucar.edu/repository/unidata-all/</url>
158156
<snapshots><enabled>false</enabled></snapshots>
159157
</repository>
160158
</repositories>
@@ -265,14 +263,6 @@
265263
</excludes>
266264
</configuration>
267265
</plugin>
268-
<plugin>
269-
<groupId>org.apache.maven.plugins</groupId>
270-
<artifactId>maven-surefire-plugin</artifactId>
271-
<version>3.0.0-M5</version>
272-
<configuration>
273-
<includes>**/*.java</includes>
274-
</configuration>
275-
</plugin>
276266
<plugin>
277267
<groupId>org.apache.maven.plugins</groupId>
278268
<artifactId>maven-gpg-plugin</artifactId>
@@ -332,4 +322,4 @@
332322
</profile>
333323
</profiles>
334324

335-
</project>
325+
</project>

‎src/main/java/com/bc/zarr/ArrayParams.java‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@
3636
* The class ArrayParams implements the Builder pattern. It is used on java side to imitate the pythonic default
3737
* value feature for function arguments. So the recognition factor for users who are familiar with the python zarr
3838
* framework should be high. E.g.:
39-
*
4039
* Python example:
4140
* <pre>
4241
* za = zarr.create(
@@ -58,7 +57,6 @@
5857
* );
5958
* </pre>
6059
* Shape must be given!
61-
*
6260
* If not given ... parameter default values are:
6361
* <pre>
6462
* boolean chunked = true;
@@ -92,7 +90,6 @@ public ArrayParams shape(int... shape) {
9290

9391
/**
9492
* Sets the optional {@code chunks} and returns a reference to this Builder so that the methods can be chained together.
95-
*
9693
* The number of dimensions must be equal to the number of dimensions of the shape.
9794
*
9895
* @param chunks the {@code chunks} to set.

‎src/main/java/com/bc/zarr/CompressorFactory.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,6 @@
4040
import java.io.OutputStream;
4141
import java.nio.ByteBuffer;
4242
import java.util.Arrays;
43-
import java.util.Collections;
4443
import java.util.HashMap;
4544
import java.util.Map;
4645
import java.util.zip.Deflater;

‎src/main/java/com/bc/zarr/ZarrArray.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -321,7 +321,14 @@ private boolean isBufferShapeEqualChunkShape(int[] bufferShape) {
321321
}
322322

323323
private boolean isZeroOffset(int[] offset) {
324-
return Arrays.equals(offset, new int[offset.length]);
324+
//return Arrays.equals(offset, new int[offset.length]);
325+
// avoid new for a test
326+
for (int i : offset) {
327+
if (i != 0) {
328+
return false;
329+
}
330+
}
331+
return true;
325332
}
326333

327334
public void writeAttributes(Map<String, Object> attributes) throws IOException {

‎src/main/java/com/bc/zarr/chunk/ChunkReaderWriterImpl_Byte.java‎

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import com.bc.zarr.storage.Store;
3030
import ucar.ma2.Array;
3131
import ucar.ma2.DataType;
32+
import ucar.ma2.IndexIterator;
3233

3334
import java.io.*;
3435

@@ -57,14 +58,31 @@ public Array read(String storeKey) throws IOException {
5758
}
5859
}
5960

61+
protected boolean isFillOnly(Array array) {
62+
if (fill == null) {
63+
return false;
64+
}
65+
final IndexIterator iter = array.getIndexIterator();
66+
while (iter.hasNext()) {
67+
if (iter.getByteNext() != fill.byteValue()) {
68+
return false;
69+
}
70+
}
71+
return true;
72+
}
73+
6074
@Override
6175
public void write(String storeKey, Array array) throws IOException {
62-
final byte[] bytes = (byte[]) array.get1DJavaArray(DataType.BYTE);
63-
try (
64-
final ByteArrayInputStream is = new ByteArrayInputStream(bytes);
65-
final OutputStream os = store.getOutputStream(storeKey)
66-
) {
67-
compressor.compress(is, os);
76+
if (isFillOnly(array)) {
77+
store.delete(storeKey);
78+
} else {
79+
final byte[] bytes = (byte[]) array.get1DJavaArray(DataType.BYTE);
80+
try (
81+
final ByteArrayInputStream is = new ByteArrayInputStream(bytes);
82+
final OutputStream os = store.getOutputStream(storeKey)
83+
) {
84+
compressor.compress(is, os);
85+
}
6886
}
6987
}
7088
}

‎src/main/java/com/bc/zarr/chunk/ChunkReaderWriterImpl_Double.java‎

Lines changed: 37 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import com.bc.zarr.storage.Store;
3030
import ucar.ma2.Array;
3131
import ucar.ma2.DataType;
32+
import ucar.ma2.IndexIterator;
3233

3334
import javax.imageio.stream.ImageInputStream;
3435
import javax.imageio.stream.ImageOutputStream;
@@ -69,18 +70,44 @@ public Array read(String storeKey) throws IOException {
6970
}
7071
}
7172

73+
protected boolean isFillOnly(Array array) {
74+
if (fill == null) {
75+
return false;
76+
}
77+
final IndexIterator iter = array.getIndexIterator();
78+
final double fillValue = fill.doubleValue();
79+
if (Double.isNaN(fillValue)) {
80+
while (iter.hasNext()) {
81+
if (! Double.isNaN(iter.getDoubleNext())) {
82+
return false;
83+
}
84+
}
85+
} else {
86+
while (iter.hasNext()) {
87+
if (iter.getDoubleNext() != fillValue) {
88+
return false;
89+
}
90+
}
91+
}
92+
return true;
93+
}
94+
7295
@Override
7396
public void write(String storeKey, Array array) throws IOException {
74-
try (
75-
final ImageOutputStream iis = new MemoryCacheImageOutputStream(new ByteArrayOutputStream());
76-
final InputStream is = new ZarrInputStreamAdapter(iis);
77-
final OutputStream os = store.getOutputStream(storeKey)
78-
) {
79-
final double[] doubles = (double[]) array.get1DJavaArray(DataType.DOUBLE);
80-
iis.setByteOrder(order);
81-
iis.writeDoubles(doubles, 0, doubles.length);
82-
iis.seek(0);
83-
compressor.compress(is, os);
97+
if (isFillOnly(array)) {
98+
store.delete(storeKey);
99+
} else {
100+
try (
101+
final ImageOutputStream iis = new MemoryCacheImageOutputStream(new ByteArrayOutputStream());
102+
final InputStream is = new ZarrInputStreamAdapter(iis);
103+
final OutputStream os = store.getOutputStream(storeKey)
104+
) {
105+
final double[] doubles = (double[]) array.get1DJavaArray(DataType.DOUBLE);
106+
iis.setByteOrder(order);
107+
iis.writeDoubles(doubles, 0, doubles.length);
108+
iis.seek(0);
109+
compressor.compress(is, os);
110+
}
84111
}
85112
}
86113
}

‎src/main/java/com/bc/zarr/chunk/ChunkReaderWriterImpl_Float.java‎

Lines changed: 37 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import com.bc.zarr.storage.Store;
3030
import ucar.ma2.Array;
3131
import ucar.ma2.DataType;
32+
import ucar.ma2.IndexIterator;
3233

3334
import javax.imageio.stream.ImageInputStream;
3435
import javax.imageio.stream.ImageOutputStream;
@@ -69,18 +70,44 @@ public Array read(String storeKey) throws IOException {
6970
}
7071
}
7172

73+
protected boolean isFillOnly(Array array) {
74+
if (fill == null) {
75+
return false;
76+
}
77+
final IndexIterator iter = array.getIndexIterator();
78+
final float fillValue = fill.floatValue();
79+
if (Float.isNaN(fillValue)) {
80+
while (iter.hasNext()) {
81+
if (! Float.isNaN(iter.getFloatNext())) {
82+
return false;
83+
}
84+
}
85+
} else {
86+
while (iter.hasNext()) {
87+
if (iter.getFloatNext() != fillValue) {
88+
return false;
89+
}
90+
}
91+
}
92+
return true;
93+
}
94+
7295
@Override
7396
public void write(String storeKey, Array array) throws IOException {
74-
try (
75-
final ImageOutputStream iis = new MemoryCacheImageOutputStream(new ByteArrayOutputStream());
76-
final InputStream is = new ZarrInputStreamAdapter(iis);
77-
final OutputStream os = store.getOutputStream(storeKey)
78-
) {
79-
final float[] floats = (float[]) array.get1DJavaArray(DataType.FLOAT);
80-
iis.setByteOrder(order);
81-
iis.writeFloats(floats, 0, floats.length);
82-
iis.seek(0);
83-
compressor.compress(is, os);
97+
if (isFillOnly(array)) {
98+
store.delete(storeKey);
99+
} else {
100+
try (
101+
final ImageOutputStream iis = new MemoryCacheImageOutputStream(new ByteArrayOutputStream());
102+
final InputStream is = new ZarrInputStreamAdapter(iis);
103+
final OutputStream os = store.getOutputStream(storeKey)
104+
) {
105+
final float[] floats = (float[]) array.get1DJavaArray(DataType.FLOAT);
106+
iis.setByteOrder(order);
107+
iis.writeFloats(floats, 0, floats.length);
108+
iis.seek(0);
109+
compressor.compress(is, os);
110+
}
84111
}
85112
}
86113
}

‎src/main/java/com/bc/zarr/chunk/ChunkReaderWriterImpl_Integer.java‎

Lines changed: 28 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import com.bc.zarr.storage.Store;
3030
import ucar.ma2.Array;
3131
import ucar.ma2.DataType;
32+
import ucar.ma2.IndexIterator;
3233

3334
import javax.imageio.stream.ImageInputStream;
3435
import javax.imageio.stream.ImageOutputStream;
@@ -69,18 +70,35 @@ public Array read(String storeKey) throws IOException {
6970
}
7071
}
7172

73+
protected boolean isFillOnly(Array array) {
74+
if (fill == null) {
75+
return false;
76+
}
77+
final IndexIterator iter = array.getIndexIterator();
78+
while (iter.hasNext()) {
79+
if (iter.getIntNext() != fill.intValue()) {
80+
return false;
81+
}
82+
}
83+
return true;
84+
}
85+
7286
@Override
7387
public void write(String storeKey, Array array) throws IOException {
74-
try (
75-
final ImageOutputStream iis = new MemoryCacheImageOutputStream(new ByteArrayOutputStream());
76-
final InputStream is = new ZarrInputStreamAdapter(iis);
77-
final OutputStream os = store.getOutputStream(storeKey)
78-
) {
79-
final int[] ints = (int[]) array.get1DJavaArray(DataType.INT);
80-
iis.setByteOrder(order);
81-
iis.writeInts(ints, 0, ints.length);
82-
iis.seek(0);
83-
compressor.compress(is, os);
88+
if (isFillOnly(array)) {
89+
store.delete(storeKey);
90+
} else {
91+
try (
92+
final ImageOutputStream iis = new MemoryCacheImageOutputStream(new ByteArrayOutputStream());
93+
final InputStream is = new ZarrInputStreamAdapter(iis);
94+
final OutputStream os = store.getOutputStream(storeKey)
95+
) {
96+
final int[] ints = (int[]) array.get1DJavaArray(DataType.INT);
97+
iis.setByteOrder(order);
98+
iis.writeInts(ints, 0, ints.length);
99+
iis.seek(0);
100+
compressor.compress(is, os);
101+
}
84102
}
85103
}
86104
}

0 commit comments

Comments
 (0)