diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 759a84d..0108c47 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,12 +36,6 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v6 - - name: Set up zarr-python - run: | - uv venv && uv init - uv add zarr - uv add zarrita - - name: Download testdata run: | mkdir testoutput diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 47fbbca..e838449 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -43,7 +43,6 @@ jobs: run: | uv venv && uv init uv add zarr - uv add zarrita - name: Download testdata run: | diff --git a/README.md b/README.md index 0f0010b..7a87b50 100644 --- a/README.md +++ b/README.md @@ -43,12 +43,7 @@ array.write( ## Development Start-Guide ### Run Tests Locally -To be able to run the tests locally, make sure to have `python3.11` installed. -Also, you need to set up a venv for zarrita at the root of the project: -`python3.11 -m venv venv_zarrita`. - -Then install zarrita there with `venv_zarrita/Scripts/pip install zarrita` -for Windows and `venv_zarrita/bin/pip install zarrita` for Linux. +To be able to run the tests locally, make sure to have `python3.11` and `uv` installed. Furthermore, you will need the `l4_sample` test data: diff --git a/src/main/java/dev/zarr/zarrjava/core/AbstractNode.java b/src/main/java/dev/zarr/zarrjava/core/AbstractNode.java new file mode 100644 index 0000000..77fa2e9 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/core/AbstractNode.java @@ -0,0 +1,15 @@ +package dev.zarr.zarrjava.core; + +import dev.zarr.zarrjava.store.StoreHandle; + +import javax.annotation.Nonnull; + +public class AbstractNode implements Node { + + @Nonnull + public final StoreHandle storeHandle; + + protected AbstractNode(@Nonnull StoreHandle storeHandle) { + this.storeHandle = storeHandle; + } +} diff --git a/src/main/java/dev/zarr/zarrjava/core/Array.java b/src/main/java/dev/zarr/zarrjava/core/Array.java index 0401b6b..44daa75 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Array.java +++ b/src/main/java/dev/zarr/zarrjava/core/Array.java @@ -1,6 +1,7 @@ package dev.zarr.zarrjava.core; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.IndexingUtils; import dev.zarr.zarrjava.utils.MultiArrayUtils; @@ -10,27 +11,73 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Arrays; import java.util.stream.Stream; -public interface Array { +public abstract class Array extends AbstractNode { - ArrayMetadata metadata(); - - StoreHandle storeHandle(); + protected CodecPipeline codecPipeline; + public abstract ArrayMetadata metadata(); - CodecPipeline codecPipeline(); + protected Array(StoreHandle storeHandle) throws ZarrException { + super(storeHandle); + } + + /** + * Opens an existing Zarr array at a specified storage location. Automatically detects the Zarr version. + * + * @param storeHandle the storage location of the Zarr array + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr array cannot be opened + */ + public static Array open(StoreHandle storeHandle) throws IOException, ZarrException { + boolean isV3 = storeHandle.resolve(ZARR_JSON).exists(); + boolean isV2 = storeHandle.resolve(ZARRAY).exists(); + if (isV3 && isV2) { + throw new ZarrException("Both Zarr v2 and v3 arrays found at the specified location."); + } else if (isV3) { + return dev.zarr.zarrjava.v3.Array.open(storeHandle); + } else if (isV2) { + return dev.zarr.zarrjava.v2.Array.open(storeHandle); + } else { + throw new ZarrException("No Zarr array found at the specified location."); + } + } + /** + * Opens an existing Zarr array at a specified storage location. Automatically detects the Zarr version. + * + * @param path the storage location of the Zarr array + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr array cannot be opened + */ + public static Array open(Path path) throws IOException, ZarrException { + return open(new StoreHandle(new FilesystemStore(path))); + } + + /** + * Opens an existing Zarr array at a specified storage location. Automatically detects the Zarr version. + * + * @param path the storage location of the Zarr array + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr array cannot be opened + */ + public static Array open(String path) throws IOException, ZarrException { + return open(Paths.get(path)); + } /** * Writes a ucar.ma2.Array into the Zarr array at a specified offset. The shape of the Zarr array * needs be large enough for the write. * - * @param offset the offset where to write the data - * @param array the data to write + * @param offset the offset where to write the data + * @param array the data to write * @param parallel utilizes parallelism if true */ - default void write(long[] offset, ucar.ma2.Array array, boolean parallel) { + public void write(long[] offset, ucar.ma2.Array array, boolean parallel) { ArrayMetadata metadata = metadata(); if (offset.length != metadata.ndim()) { throw new IllegalArgumentException("'offset' needs to have rank '" + metadata.ndim() + "'."); @@ -42,7 +89,7 @@ default void write(long[] offset, ucar.ma2.Array array, boolean parallel) { int[] shape = array.getShape(); final int[] chunkShape = metadata.chunkShape(); - Stream chunkStream = Arrays.stream(IndexingUtils.computeChunkCoords(metadata.shape(), chunkShape, offset, shape)); + Stream chunkStream = Arrays.stream(IndexingUtils.computeChunkCoords(metadata.shape, chunkShape, offset, shape)); if (parallel) { chunkStream = chunkStream.parallel(); } @@ -50,7 +97,7 @@ default void write(long[] offset, ucar.ma2.Array array, boolean parallel) { chunkCoords -> { try { final IndexingUtils.ChunkProjection chunkProjection = - IndexingUtils.computeProjection(chunkCoords, metadata.shape(), chunkShape, offset, + IndexingUtils.computeProjection(chunkCoords, metadata.shape, chunkShape, offset, shape ); @@ -82,19 +129,19 @@ default void write(long[] offset, ucar.ma2.Array array, boolean parallel) { * * @param chunkCoords The coordinates of the chunk as computed by the offset of the chunk divided * by the chunk shape. - * @param chunkArray The data to write into the chunk + * @param chunkArray The data to write into the chunk * @throws ZarrException throws ZarrException if the write fails */ - default void writeChunk(long[] chunkCoords, ucar.ma2.Array chunkArray) throws ZarrException { + public void writeChunk(long[] chunkCoords, ucar.ma2.Array chunkArray) throws ZarrException { ArrayMetadata metadata = metadata(); String[] chunkKeys = metadata.chunkKeyEncoding().encodeChunkKey(chunkCoords); - StoreHandle chunkHandle = storeHandle().resolve(chunkKeys); + StoreHandle chunkHandle = storeHandle.resolve(chunkKeys); Object parsedFillValue = metadata.parsedFillValue(); if (parsedFillValue != null && MultiArrayUtils.allValuesEqual(chunkArray, parsedFillValue)) { chunkHandle.delete(); } else { - ByteBuffer chunkBytes = codecPipeline().encode(chunkArray); + ByteBuffer chunkBytes = codecPipeline.encode(chunkArray); chunkHandle.set(chunkBytes); } } @@ -108,22 +155,21 @@ default void writeChunk(long[] chunkCoords, ucar.ma2.Array chunkArray) throws Za * @throws ZarrException throws ZarrException if the requested chunk is outside the array's domain or if the read fails */ @Nonnull - default ucar.ma2.Array readChunk(long[] chunkCoords) - throws ZarrException { + public ucar.ma2.Array readChunk(long[] chunkCoords) throws ZarrException { ArrayMetadata metadata = metadata(); if (!chunkIsInArray(chunkCoords)) { throw new ZarrException("Attempting to read data outside of the array's domain."); } final String[] chunkKeys = metadata.chunkKeyEncoding().encodeChunkKey(chunkCoords); - final StoreHandle chunkHandle = storeHandle().resolve(chunkKeys); + final StoreHandle chunkHandle = storeHandle.resolve(chunkKeys); ByteBuffer chunkBytes = chunkHandle.read(); if (chunkBytes == null) { return metadata.allocateFillValueChunk(); } - return codecPipeline().decode(chunkBytes); + return codecPipeline.decode(chunkBytes); } @@ -134,7 +180,7 @@ default ucar.ma2.Array readChunk(long[] chunkCoords) * * @param array the data to write */ - default void write(ucar.ma2.Array array) { + public void write(ucar.ma2.Array array) { write(new long[metadata().ndim()], array); } @@ -144,9 +190,9 @@ default void write(ucar.ma2.Array array) { * Utilizes no parallelism. * * @param offset the offset where to write the data - * @param array the data to write + * @param array the data to write */ - default void write(long[] offset, ucar.ma2.Array array) { + public void write(long[] offset, ucar.ma2.Array array) { write(offset, array, false); } @@ -154,10 +200,10 @@ default void write(long[] offset, ucar.ma2.Array array) { * Writes a ucar.ma2.Array into the Zarr array at the beginning of the Zarr array. The shape of * the Zarr array needs be large enough for the write. * - * @param array the data to write + * @param array the data to write * @param parallel utilizes parallelism if true */ - default void write(ucar.ma2.Array array, boolean parallel) { + public void write(ucar.ma2.Array array, boolean parallel) { write(new long[metadata().ndim()], array, parallel); } @@ -168,8 +214,8 @@ default void write(ucar.ma2.Array array, boolean parallel) { * @throws ZarrException throws ZarrException if the read fails */ @Nonnull - default ucar.ma2.Array read() throws ZarrException { - return read(new long[metadata().ndim()], Utils.toIntArray(metadata().shape())); + public ucar.ma2.Array read() throws ZarrException { + return read(new long[metadata().ndim()], Utils.toIntArray(metadata().shape)); } /** @@ -177,11 +223,11 @@ default ucar.ma2.Array read() throws ZarrException { * Utilizes no parallelism. * * @param offset the offset where to start reading - * @param shape the shape of the data to read + * @param shape the shape of the data to read * @throws ZarrException throws ZarrException if the requested data is outside the array's domain or if the read fails */ @Nonnull - default ucar.ma2.Array read(final long[] offset, final int[] shape) throws ZarrException { + public ucar.ma2.Array read(final long[] offset, final int[] shape) throws ZarrException { return read(offset, shape, false); } @@ -192,15 +238,15 @@ default ucar.ma2.Array read(final long[] offset, final int[] shape) throws ZarrE * @throws ZarrException throws ZarrException if the requested data is outside the array's domain or if the read fails */ @Nonnull - default ucar.ma2.Array read(final boolean parallel) throws ZarrException { - return read(new long[metadata().ndim()], Utils.toIntArray(metadata().shape()), parallel); + public ucar.ma2.Array read(final boolean parallel) throws ZarrException { + return read(new long[metadata().ndim()], Utils.toIntArray(metadata().shape), parallel); } - default boolean chunkIsInArray(long[] chunkCoords) { + boolean chunkIsInArray(long[] chunkCoords) { final int[] chunkShape = metadata().chunkShape(); for (int dimIdx = 0; dimIdx < metadata().ndim(); dimIdx++) { if (chunkCoords[dimIdx] < 0 - || chunkCoords[dimIdx] * chunkShape[dimIdx] >= metadata().shape()[dimIdx]) { + || chunkCoords[dimIdx] * chunkShape[dimIdx] >= metadata().shape[dimIdx]) { return false; } } @@ -210,15 +256,14 @@ default boolean chunkIsInArray(long[] chunkCoords) { /** * Reads a part of the Zarr array based on a requested offset and shape into an ucar.ma2.Array. * - * @param offset the offset where to start reading - * @param shape the shape of the data to read + * @param offset the offset where to start reading + * @param shape the shape of the data to read * @param parallel utilizes parallelism if true * @throws ZarrException throws ZarrException if the requested data is outside the array's domain or if the read fails */ @Nonnull - default ucar.ma2.Array read(final long[] offset, final int[] shape, final boolean parallel) throws ZarrException { + public ucar.ma2.Array read(final long[] offset, final int[] shape, final boolean parallel) throws ZarrException { ArrayMetadata metadata = metadata(); - CodecPipeline codecPipeline = codecPipeline(); if (offset.length != metadata.ndim()) { throw new IllegalArgumentException("'offset' needs to have rank '" + metadata.ndim() + "'."); } @@ -226,7 +271,7 @@ default ucar.ma2.Array read(final long[] offset, final int[] shape, final boolea throw new IllegalArgumentException("'shape' needs to have rank '" + metadata.ndim() + "'."); } for (int dimIdx = 0; dimIdx < metadata.ndim(); dimIdx++) { - if (offset[dimIdx] < 0 || offset[dimIdx] + shape[dimIdx] > metadata.shape()[dimIdx]) { + if (offset[dimIdx] < 0 || offset[dimIdx] + shape[dimIdx] > metadata.shape[dimIdx]) { throw new ZarrException("Requested data is outside of the array's domain."); } } @@ -238,7 +283,7 @@ default ucar.ma2.Array read(final long[] offset, final int[] shape, final boolea final ucar.ma2.Array outputArray = ucar.ma2.Array.factory(metadata.dataType().getMA2DataType(), shape); - Stream chunkStream = Arrays.stream(IndexingUtils.computeChunkCoords(metadata.shape(), chunkShape, offset, shape)); + Stream chunkStream = Arrays.stream(IndexingUtils.computeChunkCoords(metadata.shape, chunkShape, offset, shape)); if (parallel) { chunkStream = chunkStream.parallel(); } @@ -246,7 +291,7 @@ default ucar.ma2.Array read(final long[] offset, final int[] shape, final boolea chunkCoords -> { try { final IndexingUtils.ChunkProjection chunkProjection = - IndexingUtils.computeProjection(chunkCoords, metadata.shape(), chunkShape, offset, + IndexingUtils.computeProjection(chunkCoords, metadata.shape, chunkShape, offset, shape ); @@ -258,7 +303,7 @@ default ucar.ma2.Array read(final long[] offset, final int[] shape, final boolea } final String[] chunkKeys = metadata.chunkKeyEncoding().encodeChunkKey(chunkCoords); - final StoreHandle chunkHandle = storeHandle().resolve(chunkKeys); + final StoreHandle chunkHandle = storeHandle.resolve(chunkKeys); if (!chunkHandle.exists()) { return; } @@ -281,11 +326,11 @@ default ucar.ma2.Array read(final long[] offset, final int[] shape, final boolea return outputArray; } - default ArrayAccessor access() { + public ArrayAccessor access() { return new ArrayAccessor(this); } - final class ArrayAccessor { + public static final class ArrayAccessor { @Nullable long[] offset; @Nullable @@ -335,4 +380,4 @@ public void write(@Nonnull ucar.ma2.Array content) throws ZarrException { } } -} \ No newline at end of file +} diff --git a/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java index fc42294..aaa004e 100644 --- a/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java @@ -1,5 +1,7 @@ package dev.zarr.zarrjava.core; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.utils.MultiArrayUtils; import dev.zarr.zarrjava.utils.Utils; @@ -10,23 +12,40 @@ import java.nio.ByteBuffer; import java.util.Arrays; -public interface ArrayMetadata { - int ndim(); +public abstract class ArrayMetadata { - int[] chunkShape(); + public final long[] shape; - long[] shape(); + @JsonProperty("fill_value") + public final Object fillValue; + @JsonIgnore + public final Object parsedFillValue; - DataType dataType(); + public ArrayMetadata(long[] shape, Object fillValue, DataType dataType) throws ZarrException { + this.shape = shape; + this.fillValue = fillValue; + this.parsedFillValue = parseFillValue(fillValue, dataType); + } - Array allocateFillValueChunk(); + public int ndim() { + return shape.length; + } - ChunkKeyEncoding chunkKeyEncoding(); + public abstract int[] chunkShape(); - Object parsedFillValue(); + public abstract DataType dataType(); - static Object parseFillValue(Object fillValue, @Nonnull DataType dataType) + public abstract Array allocateFillValueChunk(); + + public abstract ChunkKeyEncoding chunkKeyEncoding(); + + public abstract Object parsedFillValue(); + + public static Object parseFillValue(Object fillValue, @Nonnull DataType dataType) throws ZarrException { + if (fillValue == null) { + return null; + } boolean dataTypeIsBool = dataType == dev.zarr.zarrjava.v3.DataType.BOOL || dataType == dev.zarr.zarrjava.v2.DataType.BOOL; boolean dataTypeIsByte = dataType == dev.zarr.zarrjava.v3.DataType.INT8 || dataType == dev.zarr.zarrjava.v2.DataType.INT8 || dataType == dev.zarr.zarrjava.v3.DataType.UINT8 || dataType == dev.zarr.zarrjava.v2.DataType.UINT8; boolean dataTypeIsShort = dataType == dev.zarr.zarrjava.v3.DataType.INT16 || dataType == dev.zarr.zarrjava.v2.DataType.INT16 || dataType == dev.zarr.zarrjava.v3.DataType.UINT16 || dataType == dev.zarr.zarrjava.v2.DataType.UINT16; @@ -128,7 +147,7 @@ else if (fillValueString.startsWith("0b") || fillValueString.startsWith("0x")) { throw new ZarrException("Invalid fill value '" + fillValue + "'."); } - final class CoreArrayMetadata { + public static final class CoreArrayMetadata { public final long[] shape; public final int[] chunkShape; diff --git a/src/main/java/dev/zarr/zarrjava/core/Group.java b/src/main/java/dev/zarr/zarrjava/core/Group.java new file mode 100644 index 0000000..5cff477 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/core/Group.java @@ -0,0 +1,86 @@ +package dev.zarr.zarrjava.core; + +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.StoreHandle; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Objects; +import java.util.stream.Stream; + +public abstract class Group extends AbstractNode { + + protected Group(@Nonnull StoreHandle storeHandle) { + super(storeHandle); + } + + + /** + * Opens an existing Zarr group at a specified storage location. Automatically detects the Zarr version. + * + * @param storeHandle the storage location of the Zarr group + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr group cannot be opened + */ + public static Group open(StoreHandle storeHandle) throws IOException, ZarrException { + boolean isV3 = storeHandle.resolve(ZARR_JSON).exists(); + boolean isV2 = storeHandle.resolve(ZGROUP).exists(); + if (isV3 && isV2) { + throw new ZarrException("Both Zarr v2 and v3 groups found at " + storeHandle); + } else if (isV3) { + return dev.zarr.zarrjava.v3.Group.open(storeHandle); + } else if (isV2) { + return dev.zarr.zarrjava.v2.Group.open(storeHandle); + } else { + throw new ZarrException("No Zarr group found at " + storeHandle); + } + } + + + /** + * Opens an existing Zarr group at a specified storage location. Automatically detects the Zarr version. + * + * @param path the storage location of the Zarr group + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr group cannot be opened + */ + public static Group open(Path path) throws IOException, ZarrException { + return open(new StoreHandle(new FilesystemStore(path))); + } + + /** + * Opens an existing Zarr group at a specified storage location. Automatically detects the Zarr version. + * + * @param path the storage location of the Zarr group + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr group cannot be opened + */ + public static Group open(String path) throws IOException, ZarrException { + return open(Paths.get(path)); + } + + @Nullable + public abstract Node get(String key) throws ZarrException; + + public Stream list() { + return storeHandle.list() + .map(key -> { + try { + return get(key); + } catch (ZarrException e) { + throw new RuntimeException(e); + } + }) + .filter(Objects::nonNull); + } + + public Node[] listAsArray() { + try (Stream nodeStream = list()) { + return nodeStream.toArray(Node[]::new); + } + } +} diff --git a/src/main/java/dev/zarr/zarrjava/core/GroupMetadata.java b/src/main/java/dev/zarr/zarrjava/core/GroupMetadata.java new file mode 100644 index 0000000..695bfdf --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/core/GroupMetadata.java @@ -0,0 +1,3 @@ +package dev.zarr.zarrjava.core; + +public abstract class GroupMetadata {} diff --git a/src/main/java/dev/zarr/zarrjava/core/Node.java b/src/main/java/dev/zarr/zarrjava/core/Node.java new file mode 100644 index 0000000..0b51193 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/core/Node.java @@ -0,0 +1,62 @@ +package dev.zarr.zarrjava.core; + +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.StoreHandle; + +import java.io.IOException; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.Paths; + +public interface Node { + + String ZARR_JSON = "zarr.json"; + String ZARRAY = ".zarray"; + String ZGROUP = ".zgroup"; + + /** + * Opens an existing Zarr array or group at a specified storage location. Automatically detects the Zarr version. + * + * @param storeHandle the storage location of the Zarr array + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr array cannot be opened + */ + static Node open(StoreHandle storeHandle) throws IOException, ZarrException { + boolean isV3 = storeHandle.resolve(ZARR_JSON).exists(); + boolean isV2 = storeHandle.resolve(ZARRAY).exists() || storeHandle.resolve(ZGROUP).exists(); + + if (isV3 && isV2) { + throw new ZarrException("Both Zarr v2 and v3 nodes found at " + storeHandle); + } else if (isV3) { + return dev.zarr.zarrjava.v3.Node.open(storeHandle); + } else if (isV2) { + return dev.zarr.zarrjava.v2.Node.open(storeHandle); + } else { + throw new NoSuchFileException("No Zarr node found at " + storeHandle); + } + } + + /** + * Opens an existing Zarr array or group at a specified storage location. Automatically detects the Zarr version. + * + * @param path the storage location of the Zarr array + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr array cannot be opened + */ + static Node open(Path path) throws IOException, ZarrException { + return open(new StoreHandle(new FilesystemStore(path))); + } + + /** + * Opens an existing Zarr array or group at a specified storage location. Automatically detects the Zarr version. + * + * @param path the storage location of the Zarr array + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr array cannot be opened + */ + static Node open(String path) throws IOException, ZarrException { + return open(Paths.get(path)); + } +} + diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java b/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java index 7c70b54..4ccc8f9 100644 --- a/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java @@ -2,7 +2,7 @@ import com.fasterxml.jackson.annotation.JsonValue; import dev.zarr.zarrjava.core.codec.ArrayBytesCodec; -import ucar.ma2.Array; +import ucar.ma2.*; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -11,17 +11,65 @@ public abstract class BytesCodec extends ArrayBytesCodec { protected abstract ByteOrder getByteOrder(); @Override - public Array decode(ByteBuffer chunkBytes) { - chunkBytes.order(getByteOrder()); - return Array.factory(arrayMetadata.dataType.getMA2DataType(), arrayMetadata.chunkShape, - chunkBytes); +public Array decode(ByteBuffer chunkBytes) { + chunkBytes.order(getByteOrder()); + DataType dtype = arrayMetadata.dataType.getMA2DataType(); + int[] shape = arrayMetadata.chunkShape; + + // Array.factory does not support boolean arrays directly from ByteBuffer + if (dtype == DataType.BOOLEAN) { + int size = chunkBytes.remaining(); + boolean[] bools = new boolean[size]; + for (int i = 0; i < size; i++) { + bools[i] = chunkBytes.get(i) != 0; + } + + Index index = Index.factory(shape); + return Array.factory(DataType.BOOLEAN, index, bools); } + return Array.factory(dtype, shape, chunkBytes); +} @Override public ByteBuffer encode(Array chunkArray) { - return chunkArray.getDataAsByteBuffer(getByteOrder()); - } + ByteOrder order = getByteOrder(); + + // Boolean + if (chunkArray instanceof ArrayBoolean) { + boolean[] data = (boolean[]) chunkArray.copyTo1DJavaArray(); + ByteBuffer bb = ByteBuffer.allocate(data.length); + for (boolean b : data) { + bb.put((byte) (b ? 1 : 0)); + } + bb.flip(); + return bb; + } + // Float32 + if (chunkArray instanceof ArrayFloat) { + float[] data = (float[]) chunkArray.copyTo1DJavaArray(); + ByteBuffer bb = ByteBuffer.allocate(data.length * Float.BYTES).order(order); + for (float f : data) { + bb.putFloat(f); + } + bb.flip(); + return bb; + } + + // Float64 + if (chunkArray instanceof ArrayDouble) { + double[] data = (double[]) chunkArray.copyTo1DJavaArray(); + ByteBuffer bb = ByteBuffer.allocate(data.length * Double.BYTES).order(order); + for (double d : data) { + bb.putDouble(d); + } + bb.flip(); + return bb; + } + + // All other primitive types (byte, short, int, long, char) + return chunkArray.getDataAsByteBuffer(order); +} public enum Endian { LITTLE("little"), BIG("big"); diff --git a/src/main/java/dev/zarr/zarrjava/v2/Array.java b/src/main/java/dev/zarr/zarrjava/v2/Array.java index 52d2268..638d648 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Array.java @@ -1,31 +1,34 @@ package dev.zarr.zarrjava.v2; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.core.codec.CodecPipeline; import dev.zarr.zarrjava.v2.codec.Codec; -import dev.zarr.zarrjava.v2.codec.CodecRegistry; import dev.zarr.zarrjava.v2.codec.core.BytesCodec; import javax.annotation.Nonnull; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Arrays; import java.util.function.Function; import java.util.stream.Collectors; +import static dev.zarr.zarrjava.v2.Node.makeObjectMapper; -public class Array implements dev.zarr.zarrjava.core.Array { +public class Array extends dev.zarr.zarrjava.core.Array implements Node { - static final String ZARRAY = ".zarray"; - public ArrayMetadata metadata; - public StoreHandle storeHandle; - CodecPipeline codecPipeline; + private final ArrayMetadata metadata; + + public ArrayMetadata metadata() { + return metadata; + } protected Array(StoreHandle storeHandle, ArrayMetadata arrayMetadata) throws IOException, ZarrException { - this.storeHandle = storeHandle; + super(storeHandle); this.metadata = arrayMetadata; this.codecPipeline = new CodecPipeline(Utils.concatArrays( new Codec[]{}, @@ -53,14 +56,57 @@ public static Array open(StoreHandle storeHandle) throws IOException, ZarrExcept ); } - public static ObjectMapper makeObjectMapper() { - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.registerModule(new Jdk8Module()); - objectMapper.registerSubtypes(CodecRegistry.getNamedTypes()); - return objectMapper; + /** + * Opens an existing Zarr array at a specified storage location. + * + * @param path the storage location of the Zarr array + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr array cannot be opened + */ + public static Array open(Path path) throws IOException, ZarrException { + return open(new StoreHandle(new FilesystemStore(path))); } + /** + * Opens an existing Zarr array at a specified storage location. + * + * @param path the storage location of the Zarr array + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr array cannot be opened + */ + public static Array open(String path) throws IOException, ZarrException { + return open(Paths.get(path)); + } + + /** + * Creates a new Zarr array with the provided metadata at a specified storage location. This + * method will raise an exception if a Zarr array already exists at the specified storage + * location. + * + * @param path the storage location of the Zarr array + * @param arrayMetadata the metadata of the Zarr array + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the Zarr array cannot be created + */ + public static Array create(Path path, ArrayMetadata arrayMetadata) + throws IOException, ZarrException { + return create(new StoreHandle(new FilesystemStore(path)), arrayMetadata); + } + /** + * Creates a new Zarr array with the provided metadata at a specified storage location. This + * method will raise an exception if a Zarr array already exists at the specified storage + * location. + * + * @param path the storage location of the Zarr array + * @param arrayMetadata the metadata of the Zarr array + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the Zarr array cannot be created + */ + public static Array create(String path, ArrayMetadata arrayMetadata) + throws IOException, ZarrException { + return create(Paths.get(path), arrayMetadata); + } /** * Creates a new Zarr array with the provided metadata at a specified storage location. This * method will raise an exception if a Zarr array already exists at the specified storage @@ -127,20 +173,4 @@ public String toString() { metadata.dataType ); } - - @Override - public ArrayMetadata metadata() { - return metadata; - } - - @Override - public StoreHandle storeHandle() { - return storeHandle; - } - - @Override - public CodecPipeline codecPipeline() { - return codecPipeline; - } - } diff --git a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java index d0fa0e1..dabd285 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java @@ -13,39 +13,31 @@ import javax.annotation.Nullable; -import static dev.zarr.zarrjava.core.ArrayMetadata.parseFillValue; - -public class ArrayMetadata implements dev.zarr.zarrjava.core.ArrayMetadata { +public class ArrayMetadata extends dev.zarr.zarrjava.core.ArrayMetadata { static final int ZARR_FORMAT = 2; @JsonProperty("zarr_format") public final int zarrFormat = ZARR_FORMAT; - public long[] shape; - public int[] chunks; + public final int[] chunks; @JsonProperty("dtype") - public DataType dataType; + public final DataType dataType; @JsonIgnore public final Endianness endianness; @JsonProperty("order") - public Order order; + public final Order order; @JsonProperty("dimension_separator") - public Separator dimensionSeparator; - - @JsonProperty("fill_value") - public Object fillValue; - @JsonIgnore - public final Object parsedFillValue; + public final Separator dimensionSeparator; @Nullable - public Codec[] filters; + public final Codec[] filters; @Nullable - public Codec compressor; + public final Codec compressor; @JsonIgnore public CoreArrayMetadata coreArrayMetadata; @@ -63,46 +55,39 @@ public ArrayMetadata( @Nullable @JsonProperty(value = "compressor", required = true) Codec compressor, @Nullable @JsonProperty(value = "dimension_separator") Separator dimensionSeparator ) throws ZarrException { - super(); + super(shape, fillValue, dataType); if (zarrFormat != this.zarrFormat) { throw new ZarrException( "Expected zarr format '" + this.zarrFormat + "', got '" + zarrFormat + "'."); } - this.shape = shape; this.chunks = chunks; this.dataType = dataType; this.endianness = dataType.getEndianness(); - this.fillValue = fillValue; - if (fillValue == null) { - this.parsedFillValue = null; - } else { - this.parsedFillValue = parseFillValue(fillValue, this.dataType); - } this.order = order; this.dimensionSeparator = dimensionSeparator; - this.filters = filters; - this.compressor = compressor; this.coreArrayMetadata = - new ArrayMetadata.CoreArrayMetadata(shape, chunks, + new CoreArrayMetadata(shape, chunks, this.dataType, - parsedFillValue + this.parsedFillValue ); + if (filters == null) this.filters = null; + else{ + this.filters = new Codec[filters.length]; + for(int i = 0; i < filters.length; i++) { + this.filters[i] = filters[i].evolveFromCoreArrayMetadata(this.coreArrayMetadata); + } + } + this.compressor = compressor == null ? null : compressor.evolveFromCoreArrayMetadata(this.coreArrayMetadata); } - public int ndim() { - return shape.length; - } @Override public int[] chunkShape() { return chunks; } - @Override - public long[] shape() { - return shape; - } + @Override public DataType dataType() { @@ -111,7 +96,7 @@ public DataType dataType() { @Override public Array allocateFillValueChunk() { - ucar.ma2.Array outputArray = ucar.ma2.Array.factory(dataType.getMA2DataType(), chunks); + Array outputArray = Array.factory(dataType.getMA2DataType(), chunks); if (parsedFillValue != null) MultiArrayUtils.fill(outputArray, parsedFillValue); return outputArray; } diff --git a/src/main/java/dev/zarr/zarrjava/v2/Group.java b/src/main/java/dev/zarr/zarrjava/v2/Group.java new file mode 100644 index 0000000..38c0b94 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/v2/Group.java @@ -0,0 +1,90 @@ +package dev.zarr.zarrjava.v2; + +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.utils.Utils; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.function.Function; +import static dev.zarr.zarrjava.v2.Node.makeObjectMapper; + +public class Group extends dev.zarr.zarrjava.core.Group implements Node{ + public GroupMetadata metadata; + + protected Group(@Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMetadata) throws IOException { + super(storeHandle); + this.metadata = groupMetadata; + } + + public static Group open(@Nonnull StoreHandle storeHandle) throws IOException { + StoreHandle metadataHandle = storeHandle.resolve(ZGROUP); + ByteBuffer metadataBytes = metadataHandle.readNonNull(); + return new Group(storeHandle, makeObjectMapper() + .readValue(Utils.toArray(metadataBytes), GroupMetadata.class)); + } + + public static Group open(Path path) throws IOException { + return open(new StoreHandle(new FilesystemStore(path))); + } + + public static Group open(String path) throws IOException { + return open(Paths.get(path)); + } + + public static Group create( + @Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMetadata + ) throws IOException { + ObjectMapper objectMapper = makeObjectMapper(); + ByteBuffer metadataBytes = ByteBuffer.wrap(objectMapper.writeValueAsBytes(groupMetadata)); + storeHandle.resolve(ZGROUP).set(metadataBytes); + return new Group(storeHandle, groupMetadata); + } + + public static Group create(@Nonnull StoreHandle storeHandle) throws IOException, ZarrException { + return create(storeHandle, new GroupMetadata()); + } + + public static Group create(Path path) throws IOException, ZarrException { + return create(new StoreHandle(new FilesystemStore(path))); + } + + public static Group create(String path) throws IOException, ZarrException { + return create(Paths.get(path)); + } + + @Nullable + public Node get(String key) throws ZarrException { + StoreHandle keyHandle = storeHandle.resolve(key); + try { + return Node.open(keyHandle); + } catch (IOException e) { + return null; + } + } + + public Group createGroup(String key) throws IOException, ZarrException { + return Group.create(storeHandle.resolve(key)); + } + + public Array createArray(String key, ArrayMetadata arrayMetadata) + throws IOException, ZarrException { + return Array.create(storeHandle.resolve(key), arrayMetadata); + } + + public Array createArray(String key, Function arrayMetadataBuilderMapper) + throws IOException, ZarrException { + return Array.create(storeHandle.resolve(key), arrayMetadataBuilderMapper, false); + } + + @Override + public String toString() { + return String.format("", storeHandle); + } +} diff --git a/src/main/java/dev/zarr/zarrjava/v2/GroupMetadata.java b/src/main/java/dev/zarr/zarrjava/v2/GroupMetadata.java new file mode 100644 index 0000000..e32f106 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/v2/GroupMetadata.java @@ -0,0 +1,27 @@ +package dev.zarr.zarrjava.v2; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import dev.zarr.zarrjava.ZarrException; + +public final class GroupMetadata extends dev.zarr.zarrjava.core.GroupMetadata { + + static final int ZARR_FORMAT = 2; + @JsonProperty("zarr_format") + public final int zarrFormat = ZARR_FORMAT; + + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public GroupMetadata( + @JsonProperty(value = "zarr_format", required = true) int zarrFormat + ) throws ZarrException { + if (zarrFormat != this.zarrFormat) { + throw new ZarrException( + "Expected zarr format '" + this.zarrFormat + "', got '" + zarrFormat + "'."); + } + } + + public GroupMetadata() throws ZarrException { + this(ZARR_FORMAT); + } + +} diff --git a/src/main/java/dev/zarr/zarrjava/v2/Node.java b/src/main/java/dev/zarr/zarrjava/v2/Node.java new file mode 100644 index 0000000..4623200 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/v2/Node.java @@ -0,0 +1,56 @@ +package dev.zarr.zarrjava.v2; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.v2.codec.CodecRegistry; + +import java.io.IOException; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.Paths; + +public interface Node extends dev.zarr.zarrjava.core.Node { + + static ObjectMapper makeObjectMapper() { + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.registerModule(new Jdk8Module()); + objectMapper.registerSubtypes(CodecRegistry.getNamedTypes()); + return objectMapper; + } + + /** + * Opens an existing Zarr array or group at a specified storage location. + * + * @param storeHandle the storage location of the Zarr array + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr array or group cannot be opened + */ + static Node open(StoreHandle storeHandle) throws IOException, ZarrException { + boolean isGroup = storeHandle.resolve(ZGROUP).exists(); + boolean isArray = storeHandle.resolve(ZARRAY).exists(); + + if (isGroup && isArray) { + throw new ZarrException("Store handle '" + storeHandle + "' contains both a " + ZGROUP + " and a " + ZARRAY + " file."); + } else if (isGroup) { + return Group.open(storeHandle); + } else if (isArray) { + try { + return Array.open(storeHandle); + } catch (IOException e) { + throw new ZarrException("Failed to read array metadata for store handle '" + storeHandle + "'.", e); + } + } + throw new NoSuchFileException("Store handle '" + storeHandle + "' does not contain a " + ZGROUP + " or a " + ZARRAY + " file."); + } + + static Node open(Path path) throws IOException, ZarrException { + return open(new StoreHandle(new FilesystemStore(path))); + } + + static Node open(String path) throws IOException, ZarrException { + return open(Paths.get(path)); + } +} diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/Codec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/Codec.java index 43f02f1..8c20547 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/Codec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/Codec.java @@ -1,7 +1,11 @@ package dev.zarr.zarrjava.v2.codec; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.v2.ArrayMetadata; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "id") -public interface Codec extends dev.zarr.zarrjava.core.codec.Codec {} +public interface Codec extends dev.zarr.zarrjava.core.codec.Codec { + Codec evolveFromCoreArrayMetadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException; +} diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java index 4838488..0bccc18 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java @@ -12,6 +12,7 @@ import com.fasterxml.jackson.databind.ser.std.StdSerializer; import com.scalableminds.bloscjava.Blosc; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.ArrayMetadata; import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.v2.codec.Codec; @@ -44,9 +45,6 @@ public BloscCodec( @JsonProperty(value = "typesize", defaultValue = "0") int typesize, @JsonProperty(value = "blocksize", defaultValue = "0") int blocksize ) throws ZarrException { - if (typesize < 1 && shuffle != Blosc.Shuffle.NO_SHUFFLE) { - typesize = 4; //todo: in v2 typesize is not a required parameter. default to correct value based on dtype - } if (clevel < 0 || clevel > 9) { throw new ZarrException("'clevel' needs to be between 0 and 9."); } @@ -57,7 +55,6 @@ public BloscCodec( this.blocksize = blocksize; } - @Override public ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException { @@ -72,6 +69,20 @@ public ByteBuffer encode(ByteBuffer chunkBytes) } } + @Override + public BloscCodec evolveFromCoreArrayMetadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { + if (typesize == 0) { + return new BloscCodec( + this.cname, + this.shuffle, + this.clevel, + arrayMetadata.dataType.getByteCount(), + this.blocksize + ); + } + return this; + } + public static final class CustomShuffleSerializer extends StdSerializer { public CustomShuffleSerializer() { diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BytesCodec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BytesCodec.java index ac0cd39..773022a 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BytesCodec.java @@ -2,6 +2,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.ArrayMetadata; import dev.zarr.zarrjava.v2.codec.Codec; import javax.annotation.Nonnull; @@ -22,5 +24,10 @@ public BytesCodec( protected ByteOrder getByteOrder() { return endian.getByteOrder(); } + + @Override + public Codec evolveFromCoreArrayMetadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) { + return this; + } } diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java index 8b2bed6..5093b58 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.ArrayMetadata; import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.v2.codec.Codec; import dev.zarr.zarrjava.core.codec.BytesBytesCodec; @@ -52,4 +53,9 @@ public ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException { throw new ZarrException("Error in encoding zlib.", ex); } } + + @Override + public Codec evolveFromCoreArrayMetadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) { + return this; + } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/Array.java b/src/main/java/dev/zarr/zarrjava/v3/Array.java index f21b020..4d1434f 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Array.java @@ -2,25 +2,31 @@ import com.fasterxml.jackson.databind.ObjectMapper; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.core.codec.CodecPipeline; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nonnull; +import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; -public class Array extends Node implements dev.zarr.zarrjava.core.Array { +public class Array extends dev.zarr.zarrjava.core.Array implements Node { - public ArrayMetadata metadata; - CodecPipeline codecPipeline; + private final ArrayMetadata metadata; - protected Array(StoreHandle storeHandle, ArrayMetadata arrayMetadata) - throws ZarrException { + public ArrayMetadata metadata(){ + return metadata; + } + + protected Array(StoreHandle storeHandle, ArrayMetadata arrayMetadata) throws ZarrException { super(storeHandle); this.metadata = arrayMetadata; this.codecPipeline = new CodecPipeline(arrayMetadata.codecs, arrayMetadata.coreArrayMetadata); @@ -44,6 +50,58 @@ public static Array open(StoreHandle storeHandle) throws IOException, ZarrExcept ); } + /** + * Opens an existing Zarr array at a specified storage location using a Path. + * + * @param path the storage location of the Zarr array + * @throws IOException if the metadata cannot be read + * @throws ZarrException if the Zarr array cannot be opened + */ + public static Array open(Path path) throws IOException, ZarrException { + return open(new FilesystemStore(path).resolve()); + } + + /** + * Opens an existing Zarr array at a specified storage location using a String path. + * + * @param path the storage location of the Zarr array + * @throws IOException if the metadata cannot be read + * @throws ZarrException if the Zarr array cannot be opened + */ + public static Array open(String path) throws IOException, ZarrException { + return open(Paths.get(path)); + } + + /** + * Creates a new Zarr array with the provided metadata at a specified storage location. This + * method will raise an exception if a Zarr array already exists at the specified storage + * location. + * + * @param path the storage location of the Zarr array + * @param arrayMetadata the metadata of the Zarr array + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the Zarr array cannot be created + */ + public static Array create(Path path, ArrayMetadata arrayMetadata) + throws IOException, ZarrException { + return Array.create(new FilesystemStore(path).resolve(), arrayMetadata); + } + + /** + * Creates a new Zarr array with the provided metadata at a specified storage location. This + * method will raise an exception if a Zarr array already exists at the specified storage + * location. + * + * @param path the storage location of the Zarr array + * @param arrayMetadata the metadata of the Zarr array + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the Zarr array cannot be created + */ + public static Array create(String path, ArrayMetadata arrayMetadata) + throws IOException, ZarrException { + return Array.create(Paths.get(path), arrayMetadata); + } + /** * Creates a new Zarr array with the provided metadata at a specified storage location. This * method will raise an exception if a Zarr array already exists at the specified storage @@ -103,6 +161,16 @@ public static Array create(StoreHandle storeHandle, arrayMetadataBuilderMapper.apply(new ArrayMetadataBuilder()).build(), existsOk); } + public static Array create(Path path, Function arrayMetadataBuilderMapper, boolean existsOk) + throws IOException, ZarrException { + return Array.create(new FilesystemStore(path).resolve(), arrayMetadataBuilderMapper, existsOk); + } + + public static Array create(String path, Function arrayMetadataBuilderMapper, boolean existsOk) + throws IOException, ZarrException { + return Array.create(Paths.get(path), arrayMetadataBuilderMapper, existsOk); + } + @Nonnull public static ArrayMetadataBuilder metadataBuilder() { return new ArrayMetadataBuilder(); @@ -113,21 +181,6 @@ public static ArrayMetadataBuilder metadataBuilder(ArrayMetadata existingMetadat return ArrayMetadataBuilder.fromArrayMetadata(existingMetadata); } - @Override - public CodecPipeline codecPipeline() { - return codecPipeline; - } - - - - - @Override - public ArrayMetadata metadata() { - return metadata; - } - - - private Array writeMetadata(ArrayMetadata newArrayMetadata) throws ZarrException, IOException { ObjectMapper objectMapper = makeObjectMapper(); ByteBuffer metadataBytes = ByteBuffer.wrap(objectMapper.writeValueAsBytes(newArrayMetadata)); diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java index d50d9d0..3625348 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java @@ -9,7 +9,6 @@ import dev.zarr.zarrjava.v3.chunkkeyencoding.ChunkKeyEncoding; import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.v3.codec.core.ShardingIndexedCodec; -import static dev.zarr.zarrjava.core.ArrayMetadata.parseFillValue; import java.util.Arrays; import java.util.Map; @@ -18,9 +17,8 @@ import javax.annotation.Nullable; -public final class ArrayMetadata implements dev.zarr.zarrjava.core.ArrayMetadata { - - static final String NODE_TYPE = "array"; +public final class ArrayMetadata extends dev.zarr.zarrjava.core.ArrayMetadata { + public static final String NODE_TYPE = "array"; static final int ZARR_FORMAT = 3; @JsonProperty("zarr_format") @@ -28,9 +26,6 @@ public final class ArrayMetadata implements dev.zarr.zarrjava.core.ArrayMetadata @JsonProperty("node_type") public final String nodeType = NODE_TYPE; - - public final long[] shape; - @JsonProperty("data_type") public final DataType dataType; @@ -40,11 +35,6 @@ public final class ArrayMetadata implements dev.zarr.zarrjava.core.ArrayMetadata @JsonProperty("chunk_key_encoding") public final ChunkKeyEncoding chunkKeyEncoding; - @JsonProperty("fill_value") - public final Object fillValue; - @JsonIgnore - public final Object parsedFillValue; - @JsonProperty("codecs") public final Codec[] codecs; @Nullable @@ -52,10 +42,10 @@ public final class ArrayMetadata implements dev.zarr.zarrjava.core.ArrayMetadata public final Map attributes; @Nullable @JsonProperty("dimension_names") - public String[] dimensionNames; + public final String[] dimensionNames; @Nullable @JsonProperty("storage_transformers") - public Map[] storageTransformers; + public final Map[] storageTransformers; @JsonIgnore public CoreArrayMetadata coreArrayMetadata; @@ -88,6 +78,7 @@ public ArrayMetadata( @Nullable @JsonProperty(value = "attributes") Map attributes, @Nullable @JsonProperty(value = "storage_transformers") Map[] storageTransformers ) throws ZarrException { + super(shape, fillValue, dataType); if (zarrFormat != this.zarrFormat) { throw new ZarrException( "Expected zarr format '" + this.zarrFormat + "', got '" + zarrFormat + "'."); @@ -122,22 +113,19 @@ public ArrayMetadata( shardingCodec = getShardingIndexedCodec(shardingConfig.codecs); } } - - this.shape = shape; - this.dataType = dataType; this.chunkGrid = chunkGrid; + this.dataType = dataType; + this.coreArrayMetadata = + new CoreArrayMetadata(this.shape, ((RegularChunkGrid) chunkGrid).configuration.chunkShape, + this.dataType, + this.parsedFillValue + ); + this.chunkKeyEncoding = chunkKeyEncoding; - this.fillValue = fillValue; - this.parsedFillValue = parseFillValue(fillValue, dataType); this.codecs = codecs; this.dimensionNames = dimensionNames; this.attributes = attributes; this.storageTransformers = storageTransformers; - this.coreArrayMetadata = - new CoreArrayMetadata(shape, ((RegularChunkGrid) chunkGrid).configuration.chunkShape, - dataType, - parsedFillValue - ); } @@ -155,10 +143,6 @@ public Object parsedFillValue() { return parsedFillValue; } - public int ndim() { - return shape.length; - } - public static Optional getShardingIndexedCodec(Codec[] codecs) { return Arrays.stream(codecs).filter(codec -> codec instanceof ShardingIndexedCodec).findFirst(); } @@ -167,11 +151,6 @@ public int[] chunkShape() { return ((RegularChunkGrid) this.chunkGrid).configuration.chunkShape; } - @Override - public long[] shape() { - return shape; - } - @Override public DataType dataType() { return dataType; diff --git a/src/main/java/dev/zarr/zarrjava/v3/Group.java b/src/main/java/dev/zarr/zarrjava/v3/Group.java index 2abcff3..9c56f25 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Group.java @@ -2,22 +2,25 @@ import com.fasterxml.jackson.databind.ObjectMapper; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Map; -import java.util.Objects; import java.util.function.Function; -import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; -public class Group extends Node { + +public class Group extends dev.zarr.zarrjava.core.Group implements Node { public GroupMetadata metadata; - Group(@Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMetadata) throws IOException { + protected Group(@Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMetadata) throws IOException { super(storeHandle); this.metadata = groupMetadata; } @@ -25,14 +28,23 @@ public class Group extends Node { public static Group open(@Nonnull StoreHandle storeHandle) throws IOException { StoreHandle metadataHandle = storeHandle.resolve(ZARR_JSON); ByteBuffer metadataBytes = metadataHandle.readNonNull(); - return new Group(storeHandle, Node.makeObjectMapper() + return new Group(storeHandle, makeObjectMapper() .readValue(Utils.toArray(metadataBytes), GroupMetadata.class)); } + + public static Group open(Path path) throws IOException { + return open(new StoreHandle(new FilesystemStore(path))); + } + + public static Group open(String path) throws IOException { + return open(Paths.get(path)); + } + public static Group create( @Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMetadata ) throws IOException { - ObjectMapper objectMapper = Node.makeObjectMapper(); + ObjectMapper objectMapper = makeObjectMapper(); ByteBuffer metadataBytes = ByteBuffer.wrap(objectMapper.writeValueAsBytes(groupMetadata)); storeHandle.resolve(ZARR_JSON) .set(metadataBytes); @@ -50,30 +62,27 @@ public static Group create(@Nonnull StoreHandle storeHandle) throws IOException, return create(storeHandle, GroupMetadata.defaultValue()); } + public static Group create(Path path, GroupMetadata groupMetadata) throws IOException, ZarrException { + return create(new FilesystemStore(path).resolve(), groupMetadata); + } + + public static Group create(String path, GroupMetadata groupMetadata) throws IOException, ZarrException { + return create(Paths.get(path), groupMetadata); + } + + public static Group create(Path path) throws IOException, ZarrException { + return create(new FilesystemStore(path).resolve()); + } + + public static Group create(String path) throws IOException, ZarrException { + return create(Paths.get(path)); + } + @Nullable public Node get(String key) throws ZarrException { StoreHandle keyHandle = storeHandle.resolve(key); - ObjectMapper objectMapper = Node.makeObjectMapper(); - ByteBuffer metadataBytes = keyHandle.resolve(ZARR_JSON) - .read(); - if (metadataBytes == null) { - return null; - } - byte[] metadataBytearray = Utils.toArray(metadataBytes); try { - String nodeType = objectMapper.readTree(metadataBytearray) - .get("node_type") - .asText(); - switch (nodeType) { - case ArrayMetadata.NODE_TYPE: - return new Array(keyHandle, - objectMapper.readValue(metadataBytearray, ArrayMetadata.class)); - case GroupMetadata.NODE_TYPE: - return new Group(keyHandle, - objectMapper.readValue(metadataBytearray, GroupMetadata.class)); - default: - throw new ZarrException("Unsupported node_type '" + nodeType + "' in " + keyHandle); - } + return Node.open(keyHandle); } catch (IOException e) { return null; } @@ -104,26 +113,8 @@ public Array createArray(String key, return Array.create(storeHandle.resolve(key), arrayMetadataBuilderMapper, false); } - public Stream list() { - return storeHandle.list() - .map(key -> { - try { - return get(key); - } catch (ZarrException e) { - throw new RuntimeException(e); - } - }) - .filter(Objects::nonNull); - } - - public Node[] listAsArray() { - try (Stream nodeStream = list()) { - return nodeStream.toArray(Node[]::new); - } - } - private Group writeMetadata(GroupMetadata newGroupMetadata) throws IOException { - ObjectMapper objectMapper = Node.makeObjectMapper(); + ObjectMapper objectMapper = makeObjectMapper(); ByteBuffer metadataBytes = ByteBuffer.wrap(objectMapper.writeValueAsBytes(newGroupMetadata)); storeHandle.resolve(ZARR_JSON) .set(metadataBytes); diff --git a/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java b/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java index 3b35c39..df32c04 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java @@ -7,12 +7,12 @@ import java.util.Map; import javax.annotation.Nullable; -public final class GroupMetadata { +public final class GroupMetadata extends dev.zarr.zarrjava.core.GroupMetadata { static final String NODE_TYPE = "group"; static final int ZARR_FORMAT = 3; @JsonProperty("zarr_format") - public final int zarrFormat = 3; + public final int zarrFormat = ZARR_FORMAT; @JsonProperty("node_type") public final String nodeType = "group"; diff --git a/src/main/java/dev/zarr/zarrjava/v3/Node.java b/src/main/java/dev/zarr/zarrjava/v3/Node.java index 4362999..235e37f 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Node.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Node.java @@ -2,30 +2,58 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.v3.codec.CodecRegistry; -import javax.annotation.Nonnull; -public class Node { +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.nio.file.Paths; - public final static String ZARR_JSON = "zarr.json"; - @Nonnull - public final StoreHandle storeHandle; +public interface Node extends dev.zarr.zarrjava.core.Node{ - protected Node(@Nonnull StoreHandle storeHandle) { - this.storeHandle = storeHandle; - } - - public static ObjectMapper makeObjectMapper() { + static ObjectMapper makeObjectMapper() { ObjectMapper objectMapper = new ObjectMapper(); objectMapper.registerModule(new Jdk8Module()); objectMapper.registerSubtypes(CodecRegistry.getNamedTypes()); return objectMapper; } - public StoreHandle storeHandle() { - return storeHandle; + /** + * Opens an existing Zarr array or group at a specified storage location. + * + * @param storeHandle the storage location of the Zarr array or group + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr array or group cannot be opened + */ + static Node open(StoreHandle storeHandle) throws IOException, ZarrException { + ObjectMapper objectMapper = makeObjectMapper(); + ByteBuffer metadataBytes = storeHandle.resolve(ZARR_JSON).readNonNull(); + byte[] metadataBytearray = Utils.toArray(metadataBytes); + String nodeType = objectMapper.readTree(metadataBytearray) + .get("node_type") + .asText(); + switch (nodeType) { + case ArrayMetadata.NODE_TYPE: + return new Array(storeHandle, + objectMapper.readValue(metadataBytearray, ArrayMetadata.class)); + case GroupMetadata.NODE_TYPE: + return new Group(storeHandle, + objectMapper.readValue(metadataBytearray, GroupMetadata.class)); + default: + throw new ZarrException("Unsupported node_type '" + nodeType + "' at " + storeHandle); + } } + static Node open(Path path) throws IOException, ZarrException { + return open(new StoreHandle(new FilesystemStore(path))); + } + + static Node open(String path) throws IOException, ZarrException { + return open(Paths.get(path)); + } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java index d9631a1..68ee1a5 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java @@ -5,7 +5,6 @@ import dev.zarr.zarrjava.v3.ArrayMetadata; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "name") -public interface Codec extends dev.zarr.zarrjava.v2.codec.Codec { +public interface Codec extends dev.zarr.zarrjava.core.codec.Codec { long computeEncodedSize(long inputByteLength, ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException; -} - +} \ No newline at end of file diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java index b79b1ac..75d27f5 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java @@ -28,7 +28,7 @@ public ZstdCodec( public ByteBuffer decode(ByteBuffer compressedBytes) throws ZarrException { byte[] compressedArray = compressedBytes.array(); - long originalSize = Zstd.decompressedSize(compressedArray); + long originalSize = Zstd.getFrameContentSize(compressedArray); if (originalSize == 0) { throw new ZarrException("Failed to get decompressed size"); } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java index c595005..e4e763a 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java @@ -1,50 +1,39 @@ package dev.zarr.zarrjava; +import com.github.luben.zstd.Zstd; +import com.github.luben.zstd.ZstdCompressCtx; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.v2.Group; import dev.zarr.zarrjava.v3.Array; import dev.zarr.zarrjava.v3.ArrayMetadataBuilder; import dev.zarr.zarrjava.v3.DataType; import dev.zarr.zarrjava.v3.codec.CodecBuilder; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.file.Files; +import java.io.*; +import java.nio.ByteBuffer; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; -import java.util.Comparator; import java.util.HashMap; import java.util.Map; import java.util.stream.Stream; -public class ZarrPythonTests { - final static Path TESTOUTPUT = Paths.get("testoutput"); +public class ZarrPythonTests extends ZarrTest { + final static Path PYTHON_TEST_PATH = Paths.get("src/test/python-scripts/"); - @BeforeAll - public static void clearTestoutputFolder() throws IOException { - if (Files.exists(TESTOUTPUT)) { - try (Stream walk = Files.walk(TESTOUTPUT)) { - walk.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); - } - } - Files.createDirectory(TESTOUTPUT); - } - public void run_python_script(String scriptName, String... args) throws IOException, InterruptedException { + public static int runCommand(String... command) throws IOException, InterruptedException { ProcessBuilder pb = new ProcessBuilder(); - pb.command().add("uv"); - pb.command().add("run"); - pb.command().add(PYTHON_TEST_PATH.resolve(scriptName).toString()); - pb.command().addAll(Arrays.asList(args)); + pb.command().addAll(Arrays.asList(command)); Process process = pb.start(); BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); @@ -58,60 +47,175 @@ public void run_python_script(String scriptName, String... args) throws IOExcept System.err.println(line); } - int exitCode = process.waitFor(); + return process.waitFor(); + } + + @BeforeAll + public static void setupUV() { + try { + int exitCode = runCommand("uv", "version"); + if (exitCode != 0) { + //setup uv + assert runCommand("uv", "venv") == 0; + assert runCommand("uv", "init") == 0; + assert runCommand("uv", "add", "zarr", "zstandard") == 0; + } + } catch (IOException | InterruptedException e) { + throw new RuntimeException("uv not installed or not in PATH. See"); + } + } + + public void run_python_script(String scriptName, String... args) throws IOException, InterruptedException { + int exitCode = runCommand(Stream.concat(Stream.of("uv", "run", PYTHON_TEST_PATH.resolve(scriptName) + .toString()), Arrays.stream(args)).toArray(String[]::new)); assert exitCode == 0; } + static ucar.ma2.Array testdata(dev.zarr.zarrjava.core.DataType dt){ + ucar.ma2.DataType ma2Type = dt.getMA2DataType(); + ucar.ma2.Array array = ucar.ma2.Array.factory(ma2Type, new int[]{16, 16, 16}); + for (int i = 0; i < array.getSize(); i++) { + switch (ma2Type) { + case BOOLEAN: + array.setBoolean(i, i%2 == 0); + break; + case BYTE: + case UBYTE: + array.setByte(i, (byte) i); + break; + case SHORT: + case USHORT: + array.setShort(i, (short) i); + break; + case INT: + array.setInt(i, i); + break; + case UINT: + array.setLong(i, i & 0xFFFFFFFFL); + break; + case LONG: + case ULONG: + array.setLong(i, (long) i); + break; + case FLOAT: + array.setFloat(i, (float) i); + break; + case DOUBLE: + array.setDouble(i, (double) i); + break; + default: + throw new IllegalArgumentException("Invalid DataType: " + dt); + } + } + return array; + } + + static void assertIsTestdata(ucar.ma2.Array result, dev.zarr.zarrjava.core.DataType dt) { + // expected values are i for index i + ucar.ma2.DataType ma2Type = dt.getMA2DataType(); + for (int i = 0; i < result.getSize(); i++) { + switch (ma2Type) { + case BOOLEAN: + Assertions.assertEquals(i % 2 == 0, result.getBoolean(i)); + break; + case BYTE: + case UBYTE: + Assertions.assertEquals((byte) i, result.getByte(i)); + break; + case SHORT: + case USHORT: + Assertions.assertEquals((short) i, result.getShort(i)); + break; + case INT: + Assertions.assertEquals(i, result.getInt(i)); + break; + case UINT: + Assertions.assertEquals(i & 0xFFFFFFFFL, result.getLong(i)); + break; + case LONG: + case ULONG: + Assertions.assertEquals((long) i, result.getLong(i)); + break; + case FLOAT: + Assertions.assertEquals((float) i, result.getFloat(i), 1e-6); + break; + case DOUBLE: + Assertions.assertEquals((double) i, result.getDouble(i), 1e-12); + break; + default: + throw new IllegalArgumentException("Invalid DataType: " + dt); + } + } + } + + static Stream compressorAndDataTypeProviderV3() { + Stream datatypeTests = Stream.of( +// DataType.BOOL, +// DataType.INT8, +// DataType.UINT8, // -> BUG: see https://github.com/zarr-developers/zarr-java/issues/27 + DataType.INT16, + DataType.UINT16, + DataType.INT32, + DataType.UINT32, + DataType.INT64, + DataType.UINT64, + DataType.FLOAT32, + DataType.FLOAT64 + ).flatMap(dt -> Stream.of( + new Object[]{"sharding", "end", dt}, + new Object[]{"blosc", "blosclz_shuffle_3", dt} + )); + + Stream codecsTests = Stream.of( + new Object[]{"blosc", "blosclz_noshuffle_0", DataType.INT32}, + new Object[]{"blosc", "lz4_shuffle_6", DataType.INT32}, + new Object[]{"blosc", "lz4hc_bitshuffle_3", DataType.INT32}, + new Object[]{"blosc", "zlib_shuffle_5", DataType.INT32}, + new Object[]{"blosc", "zstd_bitshuffle_9", DataType.INT32}, + new Object[]{"gzip", "0", DataType.INT32}, + new Object[]{"gzip", "5", DataType.INT32}, + new Object[]{"zstd", "0_true", DataType.INT32}, + new Object[]{"zstd", "5_true", DataType.INT32}, + new Object[]{"zstd", "0_false", DataType.INT32}, + new Object[]{"zstd", "5_false", DataType.INT32}, + new Object[]{"bytes", "BIG", DataType.INT32}, + new Object[]{"bytes", "LITTLE", DataType.INT32}, + new Object[]{"transpose", "_", DataType.INT32}, + new Object[]{"sharding", "start", DataType.INT32}, + new Object[]{"sharding_nested", "_", DataType.INT32}, + new Object[]{"crc32c", "_", DataType.INT32} + ); + + return Stream.concat(datatypeTests, codecsTests); + } + + @ParameterizedTest - @CsvSource({ - "blosc,blosclz_noshuffle_0", "blosc,lz4_shuffle_6", "blosc,lz4hc_bitshuffle_3", "blosc,zlib_shuffle_5", "blosc,zstd_bitshuffle_9", - "gzip,0", "gzip,5", - "zstd,0_true", "zstd,5_true", "zstd,0_false", "zstd,5_false", - "bytes,BIG", "bytes,LITTLE", - "transpose,_", - "sharding,start", "sharding,end", - "sharding_nested,_", - "crc32c,_", - }) - public void testReadFromZarrPythonV3(String codec, String codecParam) throws IOException, ZarrException, InterruptedException { - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("read_from_zarr_python", codec, codecParam); - run_python_script("zarr_python_write.py", codec, codecParam, storeHandle.toPath().toString()); + @MethodSource("compressorAndDataTypeProviderV3") + public void testReadV3(String codec, String codecParam, DataType dataType) throws IOException, ZarrException, InterruptedException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testReadV3", codec, codecParam, dataType.name()); + run_python_script("zarr_python_write.py", codec, codecParam, dataType.name().toLowerCase(), storeHandle.toPath().toString()); Array array = Array.open(storeHandle); ucar.ma2.Array result = array.read(); - //for expected values see zarr_python_write.py Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); - Assertions.assertEquals(DataType.INT32, array.metadata.dataType); - Assertions.assertArrayEquals(new int[]{2, 4, 8}, array.metadata.chunkShape()); - Assertions.assertEquals(42, array.metadata.attributes.get("answer")); + Assertions.assertEquals(dataType, array.metadata().dataType); + Assertions.assertArrayEquals(new int[]{2, 4, 8}, array.metadata().chunkShape()); + Assertions.assertEquals(42, array.metadata().attributes.get("answer")); - int[] expectedData = new int[16 * 16 * 16]; - Arrays.setAll(expectedData, p -> p); - Assertions.assertArrayEquals(expectedData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.INT)); + assertIsTestdata(result, DataType.INT32); } @ParameterizedTest - @CsvSource({ - "blosc,blosclz_noshuffle_0", "blosc,lz4_shuffle_6", "blosc,lz4hc_bitshuffle_3", "blosc,zlib_shuffle_5", "blosc,zstd_bitshuffle_9", - "gzip,0", "gzip,5", - "zstd,0_true", "zstd,5_true", "zstd,0_false", "zstd,5_false", - "bytes,BIG", "bytes,LITTLE", - "transpose,_", - "sharding,start", "sharding,end", - "sharding_nested,_", - "crc32c,_", - }) - public void testWriteReadWithZarrPythonV3(String codec, String codecParam) throws Exception { - int[] testData = new int[16 * 16 * 16]; - Arrays.setAll(testData, p -> p); - + @MethodSource("compressorAndDataTypeProviderV3") + public void testWriteV3(String codec, String codecParam, DataType dataType) throws Exception { Map attributes = new HashMap<>(); attributes.put("test_key", "test_value"); - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("write_to_zarr_python", codec, codecParam); + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testWriteV3", codec, codecParam, dataType.name()); ArrayMetadataBuilder builder = Array.metadataBuilder() .withShape(16, 16, 16) - .withDataType(DataType.UINT32) + .withDataType(dataType) .withChunkShape(2, 4, 8) .withFillValue(0) .withAttributes(attributes); @@ -151,64 +255,81 @@ public void testWriteReadWithZarrPythonV3(String codec, String codecParam) throw } Array writeArray = Array.create(storeHandle, builder.build()); - writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); + writeArray.write(testdata(dataType)); //read in zarr-java Array readArray = Array.open(storeHandle); ucar.ma2.Array result = readArray.read(); Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); - Assertions.assertEquals(DataType.UINT32, readArray.metadata.dataType); - Assertions.assertArrayEquals(new int[]{2, 4, 8}, readArray.metadata.chunkShape()); - Assertions.assertEquals("test_value", readArray.metadata.attributes.get("test_key")); + Assertions.assertEquals(dataType, readArray.metadata().dataType); + Assertions.assertArrayEquals(new int[]{2, 4, 8}, readArray.metadata().chunkShape()); + Assertions.assertEquals("test_value", readArray.metadata().attributes.get("test_key")); - Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + assertIsTestdata(result, DataType.INT32); //read in zarr_python - run_python_script("zarr_python_read.py", codec, codecParam, storeHandle.toPath().toString()); + run_python_script("zarr_python_read.py", codec, codecParam, dataType.name().toLowerCase(), storeHandle.toPath().toString()); } + static Stream compressorAndDataTypeProviderV2() { + Stream datatypeTests = Stream.of( + dev.zarr.zarrjava.v2.DataType.BOOL, + dev.zarr.zarrjava.v2.DataType.INT8, + dev.zarr.zarrjava.v2.DataType.UINT8, + dev.zarr.zarrjava.v2.DataType.INT16, + dev.zarr.zarrjava.v2.DataType.UINT16, + dev.zarr.zarrjava.v2.DataType.INT32, + dev.zarr.zarrjava.v2.DataType.UINT32, + dev.zarr.zarrjava.v2.DataType.INT64, + dev.zarr.zarrjava.v2.DataType.UINT64, + dev.zarr.zarrjava.v2.DataType.FLOAT32, + dev.zarr.zarrjava.v2.DataType.FLOAT64 + ).flatMap(dt -> Stream.of( + new Object[]{"zlib", "0", dt}, + new Object[]{"blosc", "blosclz_shuffle_3", dt} + )); + + Stream bloscTests = Stream.of( + new Object[]{"blosc", "blosclz_noshuffle_0", dev.zarr.zarrjava.v2.DataType.INT32}, + new Object[]{"blosc", "lz4_shuffle_6", dev.zarr.zarrjava.v2.DataType.INT32}, + new Object[]{"blosc", "lz4hc_bitshuffle_3", dev.zarr.zarrjava.v2.DataType.INT32}, + new Object[]{"blosc", "zlib_shuffle_5", dev.zarr.zarrjava.v2.DataType.INT32}, + new Object[]{"blosc", "zstd_bitshuffle_9", dev.zarr.zarrjava.v2.DataType.INT32} + ); + + return Stream.concat(datatypeTests, bloscTests); + } + @ParameterizedTest - @CsvSource({ - "zlib,0", "zlib,5", - "blosc,blosclz_noshuffle_0", "blosc,lz4_shuffle_6", "blosc,lz4hc_bitshuffle_3", "blosc,zlib_shuffle_5", "blosc,zstd_bitshuffle_9", - }) - public void testReadFromZarrPythonV2(String compressor, String compressorParam) throws IOException, ZarrException, InterruptedException { - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("read_from_zarr_python_v2", compressor, compressorParam); - run_python_script("zarr_python_write_v2.py", compressor, compressorParam, storeHandle.toPath().toString()); + @MethodSource("compressorAndDataTypeProviderV2") + public void testReadV2(String compressor, String compressorParam, dev.zarr.zarrjava.v2.DataType dt) throws IOException, ZarrException, InterruptedException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testReadV2", compressor, compressorParam, dt.name()); + run_python_script("zarr_python_write_v2.py", compressor, compressorParam, dt.name().toLowerCase(), storeHandle.toPath().toString()); dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.open(storeHandle); ucar.ma2.Array result = array.read(); - //for expected values see zarr_python_write.py Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); - Assertions.assertEquals(dev.zarr.zarrjava.v2.DataType.INT32, array.metadata.dataType); - Assertions.assertArrayEquals(new int[]{2, 4, 8}, array.metadata.chunkShape()); -// Assertions.assertEquals(42, array.metadata.attributes.get("answer")); + Assertions.assertEquals(dt, array.metadata().dataType); + Assertions.assertArrayEquals(new int[]{2, 4, 8}, array.metadata().chunkShape()); +// Assertions.assertEquals(42, array.metadata().attributes.get("answer")); - int[] expectedData = new int[16 * 16 * 16]; - Arrays.setAll(expectedData, p -> p); - Assertions.assertArrayEquals(expectedData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.INT)); + assertIsTestdata(result, dt); } @ParameterizedTest - @CsvSource({ - "zlib,0", "zlib,5", - "blosc,blosclz_noshuffle_0", "blosc,lz4_shuffle_6", "blosc,lz4hc_bitshuffle_3", "blosc,zlib_shuffle_5", "blosc,zstd_bitshuffle_9", - }) - public void testWriteReadWithZarrPythonV2(String compressor, String compressorParam) throws Exception { - int[] testData = new int[16 * 16 * 16]; - Arrays.setAll(testData, p -> p); - + @MethodSource("compressorAndDataTypeProviderV2") + public void testWriteV2(String compressor, String compressorParam, dev.zarr.zarrjava.v2.DataType dt) throws Exception { // Map attributes = new HashMap<>(); // attributes.put("test_key", "test_value"); - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("write_to_zarr_python_v2", compressor, compressorParam); + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCodecsWriteV2", compressor, compressorParam, dt.name()); dev.zarr.zarrjava.v2.ArrayMetadataBuilder builder = dev.zarr.zarrjava.v2.Array.metadataBuilder() .withShape(16, 16, 16) - .withDataType(dev.zarr.zarrjava.v2.DataType.UINT32) + .withDataType(dt) .withChunks(2, 4, 8) // .withAttributes(attributes) .withFillValue(0); @@ -228,20 +349,79 @@ public void testWriteReadWithZarrPythonV2(String compressor, String compressorPa } dev.zarr.zarrjava.v2.Array writeArray = dev.zarr.zarrjava.v2.Array.create(storeHandle, builder.build()); - writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); + writeArray.write(testdata(dt)); //read in zarr-java dev.zarr.zarrjava.v2.Array readArray = dev.zarr.zarrjava.v2.Array.open(storeHandle); ucar.ma2.Array result = readArray.read(); Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); - Assertions.assertEquals(dev.zarr.zarrjava.v2.DataType.UINT32, readArray.metadata.dataType); - Assertions.assertArrayEquals(new int[]{2, 4, 8}, readArray.metadata.chunkShape()); + Assertions.assertEquals(dt, readArray.metadata().dataType); + Assertions.assertArrayEquals(new int[]{2, 4, 8}, readArray.metadata().chunkShape()); // Assertions.assertEquals("test_value", readArray.metadata.attributes.get("test_key")); - - Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + assertIsTestdata(result, dt); //read in zarr_python - run_python_script("zarr_python_read_v2.py", compressor, compressorParam, storeHandle.toPath().toString()); + run_python_script("zarr_python_read_v2.py", compressor, compressorParam, dt.name().toLowerCase(), storeHandle.toPath().toString()); + } + + @CsvSource({"0,true", "0,false", "5, true", "10, false"}) + @ParameterizedTest + public void testZstdLibrary(int clevel, boolean checksumFlag) throws IOException, InterruptedException { + //compress using ZstdCompressCtx + int number = 123456; + byte[] src = ByteBuffer.allocate(4).putInt(number).array(); + byte[] compressed; + try (ZstdCompressCtx ctx = new ZstdCompressCtx()) { + ctx.setLevel(clevel); + ctx.setChecksum(checksumFlag); + compressed = ctx.compress(src); + } + //decompress with Zstd.decompress + long originalSize = Zstd.getFrameContentSize(compressed); + byte[] decompressed = Zstd.decompress(compressed, (int) originalSize); + Assertions.assertEquals(number, ByteBuffer.wrap(decompressed).getInt()); + + //write compressed to file + String compressedDataPath = TESTOUTPUT.resolve("compressed" + clevel + checksumFlag + ".bin").toString(); + try (FileOutputStream fos = new FileOutputStream(compressedDataPath)) { + fos.write(compressed); + } + + //decompress in python + int exitCode = ZarrPythonTests.runCommand( + "uv", + "run", + PYTHON_TEST_PATH.resolve("zstd_decompress.py").toString(), + compressedDataPath, + Integer.toString(number) + ); + assert exitCode == 0; + } + + @Test + public void testGroupReadWriteV2() throws Exception { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("group_write"); + StoreHandle storeHandle2 = new FilesystemStore(TESTOUTPUT).resolve("group_read"); + Group group = Group.create(storeHandle); + dev.zarr.zarrjava.v2.DataType dataType = dev.zarr.zarrjava.v2.DataType.INT32; + dev.zarr.zarrjava.v2.Array array = group.createGroup("group").createArray("array", arrayMetadataBuilder -> arrayMetadataBuilder + .withShape(16, 16, 16) + .withDataType(dataType) + .withChunks(2, 4, 8) + ); + + array.write(testdata(dataType)); + + run_python_script("zarr_python_group_v2.py", storeHandle.toPath().toString(), storeHandle2.toPath().toString()); + + Group group2 = Group.open(storeHandle2); + Group subgroup = (Group) group2.get("group2"); + Assertions.assertNotNull(subgroup); + dev.zarr.zarrjava.v2.Array array2 = (dev.zarr.zarrjava.v2.Array) subgroup.get("array2"); + Assertions.assertNotNull(array2); + ucar.ma2.Array result = array2.read(); + Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); + assertIsTestdata(result, dataType); } } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java new file mode 100644 index 0000000..66b0238 --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java @@ -0,0 +1,76 @@ +package dev.zarr.zarrjava; + +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.HttpStore; +import dev.zarr.zarrjava.store.S3Store; +import dev.zarr.zarrjava.v3.*; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; + +import java.io.IOException; +import java.nio.file.Files; + +import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; + +public class ZarrStoreTest extends ZarrTest { + @Test + public void testFileSystemStores() throws IOException, ZarrException { + FilesystemStore fsStore = new FilesystemStore(TESTDATA); + ObjectMapper objectMapper = makeObjectMapper(); + + GroupMetadata groupMetadata = objectMapper.readValue( + Files.readAllBytes(TESTDATA.resolve("l4_sample").resolve("zarr.json")), + GroupMetadata.class + ); + + String groupMetadataString = objectMapper.writeValueAsString(groupMetadata); + Assertions.assertTrue(groupMetadataString.contains("\"zarr_format\":3")); + Assertions.assertTrue(groupMetadataString.contains("\"node_type\":\"group\"")); + + ArrayMetadata arrayMetadata = objectMapper.readValue(Files.readAllBytes(TESTDATA.resolve( + "l4_sample").resolve("color").resolve("1").resolve("zarr.json")), + ArrayMetadata.class); + + String arrayMetadataString = objectMapper.writeValueAsString(arrayMetadata); + Assertions.assertTrue(arrayMetadataString.contains("\"zarr_format\":3")); + Assertions.assertTrue(arrayMetadataString.contains("\"node_type\":\"array\"")); + Assertions.assertTrue(arrayMetadataString.contains("\"shape\":[1,4096,4096,2048]")); + + Assertions.assertInstanceOf(Array.class, Array.open(fsStore.resolve("l4_sample", "color", "1"))); + + Node[] subNodes = Group.open(fsStore.resolve("l4_sample")).list().toArray(Node[]::new); + Assertions.assertEquals(2, subNodes.length); + Assertions.assertInstanceOf(Group.class, subNodes[0]); + + Array[] colorSubNodes = ((Group) Group.open(fsStore.resolve("l4_sample")).get("color")).list().toArray(Array[]::new); + + Assertions.assertEquals(5, colorSubNodes.length); + Assertions.assertInstanceOf(Array.class, colorSubNodes[0]); + + Array array = (Array) ((Group) Group.open(fsStore.resolve("l4_sample")).get("color")).get("1"); + Assertions.assertArrayEquals(new long[]{1, 4096, 4096, 2048}, array.metadata().shape); + } + + @Test + public void testS3Store() throws IOException, ZarrException { + S3Store s3Store = new S3Store(S3Client.builder() + .region(Region.of("eu-west-1")) + .credentialsProvider(AnonymousCredentialsProvider.create()) + .build(), "static.webknossos.org", "data"); + Array array = Array.open(s3Store.resolve("zarr_v3", "l4_sample", "color", "1")); + + Assertions.assertArrayEquals(new long[]{1, 4096, 4096, 2048}, array.metadata().shape); + } + + @Test + public void testHttpStore() throws IOException, ZarrException { + HttpStore httpStore = new dev.zarr.zarrjava.store.HttpStore("https://static.webknossos.org/data/zarr_v3/l4_sample"); + Array array = Array.open(httpStore.resolve("color", "1")); + + Assertions.assertArrayEquals(new long[]{1, 4096, 4096, 2048}, array.metadata().shape); + } +} diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java index 3627480..b40d516 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java @@ -1,612 +1,33 @@ package dev.zarr.zarrjava; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.github.luben.zstd.Zstd; -import com.github.luben.zstd.ZstdCompressCtx; -import dev.zarr.zarrjava.store.*; -import dev.zarr.zarrjava.utils.MultiArrayUtils; -import dev.zarr.zarrjava.v3.*; -import dev.zarr.zarrjava.v3.codec.CodecBuilder; -import dev.zarr.zarrjava.v3.codec.core.BytesCodec; -import dev.zarr.zarrjava.v3.codec.core.TransposeCodec; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; -import org.junit.jupiter.params.provider.MethodSource; -import org.junit.jupiter.params.provider.ValueSource; -import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.s3.S3Client; -import ucar.ma2.MAMath; - -import java.io.*; -import java.nio.ByteBuffer; +import java.io.File; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Arrays; import java.util.Comparator; -import java.util.HashMap; -import java.util.Map; -import java.util.function.Function; import java.util.stream.Stream; -import static dev.zarr.zarrjava.core.ArrayMetadata.parseFillValue; -import static org.junit.Assert.assertThrows; - public class ZarrTest { - final static Path TESTDATA = Paths.get("testdata"); - final static Path TESTOUTPUT = Paths.get("testoutput"); - final static Path PYTHON_TEST_PATH = Paths.get("src/test/python-scripts/"); + public static final Path TESTDATA = Paths.get("testdata"); + public static final Path TESTOUTPUT = Paths.get("testoutput"); @BeforeAll public static void clearTestoutputFolder() throws IOException { if (Files.exists(TESTOUTPUT)) { try (Stream walk = Files.walk(TESTOUTPUT)) { - walk.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); + walk.sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(file -> { + if (!file.delete()) { + throw new RuntimeException("Failed to delete file: " + file.getAbsolutePath()); + } + }); } } Files.createDirectory(TESTOUTPUT); } - - @CsvSource({"0,true", "0,false", "5, true", "10, false"}) - @ParameterizedTest - public void testZstdLibrary(int clevel, boolean checksumFlag) throws IOException, InterruptedException { - //compress using ZstdCompressCtx - int number = 123456; - byte[] src = ByteBuffer.allocate(4).putInt(number).array(); - byte[] compressed; - try (ZstdCompressCtx ctx = new ZstdCompressCtx()) { - ctx.setLevel(clevel); - ctx.setChecksum(checksumFlag); - compressed = ctx.compress(src); - } - //decompress with Zstd.decompress - long originalSize = Zstd.decompressedSize(compressed); - byte[] decompressed = Zstd.decompress(compressed, (int) originalSize); - Assertions.assertEquals(number, ByteBuffer.wrap(decompressed).getInt()); - - //write compressed to file - String compressedDataPath = TESTOUTPUT.resolve("compressed" + clevel + checksumFlag + ".bin").toString(); - try (FileOutputStream fos = new FileOutputStream(compressedDataPath)) { - fos.write(compressed); - } - - //decompress in python - Process process = new ProcessBuilder( - "uv", - "run", - PYTHON_TEST_PATH.resolve("zstd_decompress.py").toString(), - compressedDataPath, - Integer.toString(number) - ).start(); - int exitCode = process.waitFor(); - assert exitCode == 0; - } - - - static Stream> invalidCodecBuilder() { - return Stream.of( - c -> c.withBytes(BytesCodec.Endian.LITTLE).withBytes(BytesCodec.Endian.LITTLE), - c -> c.withBlosc().withBytes(BytesCodec.Endian.LITTLE), - c -> c.withBytes(BytesCodec.Endian.LITTLE).withTranspose(new int[]{1, 0}), - c -> c.withTranspose(new int[]{1, 0}).withBytes(BytesCodec.Endian.LITTLE).withTranspose(new int[]{1, 0}) - ); - } - - @ParameterizedTest - @MethodSource("invalidCodecBuilder") - public void testCheckInvalidCodecConfiguration(Function codecBuilder) throws Exception { - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("invalid_codec_config", String.valueOf(codecBuilder.hashCode())); - ArrayMetadataBuilder builder = Array.metadataBuilder() - .withShape(new long[]{4, 4}) - .withDataType(DataType.UINT32) - .withChunkShape(new int[]{2, 2}) - .withCodecs(codecBuilder); - - assertThrows(ZarrException.class, () -> Array.create(storeHandle, builder.build())); - } - - @Test - public void testLargerChunkSizeThanArraySize() throws ZarrException, IOException { - int[] testData = new int[16 * 16 * 16]; - Arrays.setAll(testData, p -> p); - - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("larger_chunk_size_than_array"); - ArrayMetadata metadata = Array.metadataBuilder() - .withShape(16, 16, 16) - .withDataType(DataType.UINT32) - .withChunkShape(32, 32, 32) - .withFillValue(0) - .build(); - Array writeArray = Array.create(storeHandle, metadata); - writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); - - //read in zarr-java - Array readArray = Array.open(storeHandle); - ucar.ma2.Array result = readArray.read(); - - Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); - } - - static Stream invalidChunkSizes() { - return Stream.of( - new int[]{1}, - new int[]{1, 1, 1} - ); - } - - @ParameterizedTest - @MethodSource("invalidChunkSizes") - public void testCheckInvalidChunkDimensions(int[] chunkSize) { - long[] shape = new long[]{4, 4}; - - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("invalid_chunksize"); - ArrayMetadataBuilder builder = Array.metadataBuilder() - .withShape(shape) - .withDataType(DataType.UINT32) - .withChunkShape(chunkSize); - - assertThrows(ZarrException.class, builder::build); - } - - static Stream invalidShardSizes() { - return Stream.of( - new int[]{4}, //wrong dims - new int[]{4, 4, 4}, //wrong dims - new int[]{1, 1}, //smaller than inner chunk shape - new int[]{5, 5}, //no exact multiple of inner chunk shape - new int[]{2, 1}, //smaller than inner chunk shape in 2nd dimension - new int[]{2, 5} //no exact multiple of inner chunk shape in 2nd dimension - ); - } - - @ParameterizedTest - @MethodSource("invalidShardSizes") - public void testCheckShardingBounds(int[] shardSize) throws Exception { - long[] shape = new long[]{10, 10}; - int[] innerChunkSize = new int[]{2, 2}; - - ArrayMetadataBuilder builder = Array.metadataBuilder() - .withShape(shape) - .withDataType(DataType.UINT32).withChunkShape(shardSize); - - if (false) { - int[] nestedChunkSize = new int[]{4, 4}; - builder = builder.withCodecs(c -> c.withSharding(new int[]{2, 2}, c1 -> c1.withSharding(nestedChunkSize, c2 -> c2.withBytes("LITTLE")))); - } - builder = builder.withCodecs(c -> c.withSharding(innerChunkSize, c1 -> c1.withBytes("LITTLE"))); - assertThrows(ZarrException.class, builder::build); - } - - @ParameterizedTest - @CsvSource({"0,true", "0,false", "5, true", "5, false"}) - public void testZstdCodecReadWrite(int clevel, boolean checksum) throws ZarrException, IOException { - int[] testData = new int[16 * 16 * 16]; - Arrays.setAll(testData, p -> p); - - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testZstdCodecReadWrite", "checksum_" + checksum, "clevel_" + clevel); - ArrayMetadataBuilder builder = Array.metadataBuilder() - .withShape(16, 16, 16) - .withDataType(DataType.UINT32) - .withChunkShape(2, 4, 8) - .withFillValue(0) - .withCodecs(c -> c.withZstd(clevel, checksum)); - Array writeArray = Array.create(storeHandle, builder.build()); - writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); - - Array readArray = Array.open(storeHandle); - ucar.ma2.Array result = readArray.read(); - - Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); - } - - @Test - public void testTransposeCodec() throws ZarrException { - ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{2, 3, 3}, new int[]{ - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}); - ucar.ma2.Array testDataTransposed120 = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{3, 3, 2}, new int[]{ - 0, 9, 1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7, 16, 8, 17}); - - TransposeCodec transposeCodec = new TransposeCodec(new TransposeCodec.Configuration(new int[]{1, 2, 0})); - transposeCodec.setCoreArrayMetadata(new ArrayMetadata.CoreArrayMetadata( - new long[]{2, 3, 3}, - new int[]{2, 3, 3}, - DataType.UINT32, - null)); - - assert MAMath.equals(testDataTransposed120, transposeCodec.encode(testData)); - assert MAMath.equals(testData, transposeCodec.decode(testDataTransposed120)); - } - - static Stream invalidTransposeOrder() { - return Stream.of( - new int[]{1, 0, 0}, - new int[]{1, 2, 3}, - new int[]{1, 2, 3, 0}, - new int[]{1, 2} - ); - } - - @ParameterizedTest - @MethodSource("invalidChunkSizes") - public void testCheckInvalidTransposeOrder(int[] transposeOrder) throws Exception { - int[] shapeInt = new int[]{2, 3, 3}; - long[] shapeLong = new long[]{2, 3, 3}; - - TransposeCodec transposeCodec = new TransposeCodec(new TransposeCodec.Configuration(transposeOrder)); - transposeCodec.setCoreArrayMetadata(new ArrayMetadata.CoreArrayMetadata( - shapeLong, - shapeInt, - DataType.UINT32, - null)); - - ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, shapeInt); - assertThrows(ZarrException.class, () -> transposeCodec.encode(testData)); - } - - @Test - public void testFileSystemStores() throws IOException, ZarrException { - FilesystemStore fsStore = new FilesystemStore(TESTDATA); - ObjectMapper objectMapper = Node.makeObjectMapper(); - - GroupMetadata group = objectMapper.readValue( - Files.readAllBytes(TESTDATA.resolve("l4_sample").resolve("zarr.json")), - GroupMetadata.class - ); - - System.out.println(group); - System.out.println(objectMapper.writeValueAsString(group)); - - ArrayMetadata arrayMetadata = objectMapper.readValue(Files.readAllBytes(TESTDATA.resolve( - "l4_sample").resolve("color").resolve("1").resolve("zarr.json")), - ArrayMetadata.class); - - System.out.println(arrayMetadata); - System.out.println(objectMapper.writeValueAsString(arrayMetadata)); - - System.out.println( - Array.open(fsStore.resolve("l4_sample", "color", "1"))); - System.out.println( - Arrays.toString(Group.open(fsStore.resolve("l4_sample")).list().toArray(Node[]::new))); - System.out.println( - Arrays.toString(((Group) Group.open(fsStore.resolve("l4_sample")).get("color")).list() - .toArray(Node[]::new))); - } - - @Test - public void testS3Store() throws IOException, ZarrException { - S3Store s3Store = new S3Store(S3Client.builder() - .region(Region.of("eu-west-1")) - .credentialsProvider(AnonymousCredentialsProvider.create()) - .build(), "static.webknossos.org", "data"); - System.out.println(Array.open(s3Store.resolve("zarr_v3", "l4_sample", "color", "1"))); - } - - @Test - public void testV3ShardingReadCutout() throws IOException, ZarrException { - Array array = Array.open(new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1")); - - ucar.ma2.Array outArray = array.read(new long[]{0, 3073, 3073, 513}, new int[]{1, 64, 64, 64}); - Assertions.assertEquals(outArray.getSize(), 64 * 64 * 64); - Assertions.assertEquals(outArray.getByte(0), -98); - } - - @Test - public void testV3Access() throws IOException, ZarrException { - Array readArray = Array.open(new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1")); - - ucar.ma2.Array outArray = readArray.access().withOffset(0, 3073, 3073, 513) - .withShape(1, 64, 64, 64) - .read(); - Assertions.assertEquals(outArray.getSize(), 64 * 64 * 64); - Assertions.assertEquals(outArray.getByte(0), -98); - - Array writeArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("l4_sample_2", "color", "1"), - readArray.metadata - ); - writeArray.access().withOffset(0, 3073, 3073, 513).write(outArray); - } - - @ParameterizedTest - @ValueSource(strings = {"start", "end"}) - public void testV3ShardingReadWrite(String indexLocation) throws IOException, ZarrException { - Array readArray = Array.open( - new FilesystemStore(TESTDATA).resolve("sharding_index_location", indexLocation)); - ucar.ma2.Array readArrayContent = readArray.read(); - Array writeArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("sharding_index_location", indexLocation), - readArray.metadata - ); - writeArray.write(readArrayContent); - ucar.ma2.Array outArray = writeArray.read(); - - assert MultiArrayUtils.allValuesEqual(readArrayContent, outArray); - } - - @Test - public void testV3Codecs() throws IOException, ZarrException { - int[] readShape = new int[]{1, 1, 1024, 1024}; - Array readArray = Array.open( - new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "8-8-2")); - ucar.ma2.Array readArrayContent = readArray.read(new long[4], readShape); - { - Array gzipArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("l4_sample_gzip", "color", "8-8-2"), - Array.metadataBuilder(readArray.metadata).withCodecs(c -> c.withGzip(5)).build() - ); - gzipArray.write(readArrayContent); - ucar.ma2.Array outGzipArray = gzipArray.read(new long[4], readShape); - assert MultiArrayUtils.allValuesEqual(outGzipArray, readArrayContent); - } - { - Array bloscArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("l4_sample_blosc", "color", "8-8-2"), - Array.metadataBuilder(readArray.metadata).withCodecs(c -> c.withBlosc("zstd", 5)).build() - ); - bloscArray.write(readArrayContent); - ucar.ma2.Array outBloscArray = bloscArray.read(new long[4], readShape); - assert MultiArrayUtils.allValuesEqual(outBloscArray, readArrayContent); - } - { - Array zstdArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("l4_sample_zstd", "color", "8-8-2"), - Array.metadataBuilder(readArray.metadata).withCodecs(c -> c.withZstd(10)).build() - ); - zstdArray.write(readArrayContent); - ucar.ma2.Array outZstdArray = zstdArray.read(new long[4], readShape); - assert MultiArrayUtils.allValuesEqual(outZstdArray, readArrayContent); - } - } - - @Test - public void testV3ArrayMetadataBuilder() throws ZarrException { - Array.metadataBuilder() - .withShape(1, 4096, 4096, 1536) - .withDataType(DataType.UINT32) - .withChunkShape(1, 1024, 1024, 1024) - .withFillValue(0) - .withCodecs( - c -> c.withSharding(new int[]{1, 32, 32, 32}, CodecBuilder::withBlosc)) - .build(); - } - - @Test - public void testV3FillValue() throws ZarrException { - Assertions.assertEquals((int) parseFillValue(0, DataType.UINT32), 0); - Assertions.assertEquals((int) parseFillValue("0x00010203", DataType.UINT32), 50462976); - Assertions.assertEquals((byte) parseFillValue("0b00000010", DataType.UINT8), 2); - assert Double.isNaN((double) parseFillValue("NaN", DataType.FLOAT64)); - assert Double.isInfinite((double) parseFillValue("-Infinity", DataType.FLOAT64)); - } - - @Test - public void testV3Group() throws IOException, ZarrException { - FilesystemStore fsStore = new FilesystemStore(TESTOUTPUT); - - Map attributes = new HashMap<>(); - attributes.put("hello", "world"); - - Group group = Group.create(fsStore.resolve("testgroup")); - Group group2 = group.createGroup("test2", attributes); - Array array = group2.createArray("array", b -> - b.withShape(10, 10) - .withDataType(DataType.UINT8) - .withChunkShape(5, 5) - ); - array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); - - Assertions.assertArrayEquals(((Array) ((Group) group.listAsArray()[0]).listAsArray()[0]).metadata.chunkShape(), new int[]{5, 5}); - } - - @Test - public void testReadme1() throws IOException, ZarrException { - Group hierarchy = Group.open( - new HttpStore("https://static.webknossos.org/data/zarr_v3") - .resolve("l4_sample") - ); - Group color = (Group) hierarchy.get("color"); - Array array = (Array) color.get("1"); - ucar.ma2.Array outArray = array.read( - new long[]{0, 3073, 3073, 513}, // offset - new int[]{1, 64, 64, 64} // shape - ); - } - - @Test - public void testReadme2() throws IOException, ZarrException { - Array array = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("testoutput", "color", "1"), - Array.metadataBuilder() - .withShape(1, 4096, 4096, 1536) - .withDataType(DataType.UINT32) - .withChunkShape(1, 1024, 1024, 1024) - .withFillValue(0) - .withCodecs(c -> c.withSharding(new int[]{1, 32, 32, 32}, c1 -> c1.withBlosc())) - .build() - ); - ucar.ma2.Array data = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1, 1, 2, 2}, new int[]{1, 2, 3, 4}); - array.write( - new long[]{0, 0, 0, 0}, // offset - data - ); - ucar.ma2.Array output = array.read(new long[]{0, 0, 0, 0}, new int[]{1, 1, 2, 2}); - assert MultiArrayUtils.allValuesEqual(data, output); - - } - - @ParameterizedTest - @ValueSource(strings = {"1", "2-2-1", "4-4-1", "16-16-4"}) - public void testReadL4Sample(String mag) throws IOException, ZarrException { - StoreHandle httpStoreHandle = new HttpStore("https://static.webknossos.org/data/zarr_v3/").resolve("l4_sample", "color", mag); - StoreHandle localStoreHandle = new FilesystemStore(TESTDATA).resolve("l4_sample", "color", mag); - - Array httpArray = Array.open(httpStoreHandle); - Array localArray = Array.open(localStoreHandle); - System.out.println(httpArray); - System.out.println(localArray); - - ucar.ma2.Array httpData1 = httpArray.read(new long[]{0, 0, 0, 0}, new int[]{1, 64, 64, 64}); - ucar.ma2.Array localData1 = localArray.read(new long[]{0, 0, 0, 0}, new int[]{1, 64, 64, 64}); - - assert MultiArrayUtils.allValuesEqual(httpData1, localData1); - - //offset to where l4_sample contains non-zero values - long[] offset = new long[4]; - long[] originalOffset = new long[]{0, 3073, 3073, 513}; - long[] originalShape = new long[]{1, 4096, 4096, 2048}; - long[] arrayShape = httpArray.metadata.shape; - for (int i = 0; i < 4; i++) { - offset[i] = originalOffset[i] / (originalShape[i] / arrayShape[i]); - } - - ucar.ma2.Array httpData2 = httpArray.read(offset, new int[]{1, 64, 64, 64}); - ucar.ma2.Array localData2 = localArray.read(offset, new int[]{1, 64, 64, 64}); - - assert MultiArrayUtils.allValuesEqual(httpData2, localData2); - } - - @ParameterizedTest - @ValueSource(booleans = {false, true}) - public void testParallel(boolean useParallel) throws IOException, ZarrException { - int[] testData = new int[512 * 512 * 512]; - Arrays.setAll(testData, p -> p); - - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testParallelRead"); - ArrayMetadata metadata = Array.metadataBuilder() - .withShape(512, 512, 512) - .withDataType(DataType.UINT32) - .withChunkShape(100, 100, 100) - .withFillValue(0) - .build(); - Array writeArray = Array.create(storeHandle, metadata); - writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{512, 512, 512}, testData), useParallel); - - Array readArray = Array.open(storeHandle); - ucar.ma2.Array result = readArray.read(useParallel); - - Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); - clearTestoutputFolder(); - } - - @Test - public void testMetadataAcceptsEmptyStorageTransformer() throws ZarrException, IOException { - // non-empty storage transformers are currently not supported - - Map[] storageTransformersEmpty = Array.open( - new FilesystemStore(TESTDATA).resolve("storage_transformer", "empty") - ).metadata.storageTransformers; - assert storageTransformersEmpty.length == 0; - - assertThrows(JsonMappingException.class, () -> Array.open( - new FilesystemStore(TESTDATA).resolve("storage_transformer", "exists")) - ); - - ArrayMetadataBuilder builderWithStorageTransformer = Array.metadataBuilder() - .withShape(1) - .withChunkShape(1) - .withDataType(DataType.UINT8) - .withStorageTransformers(new HashMap[]{new HashMap() {{ - put("some", "value"); - }}}); - - assertThrows(ZarrException.class, () -> Array.create( - new FilesystemStore(TESTOUTPUT).resolve("storage_transformer"), - builderWithStorageTransformer.build() - )); - } - - @ParameterizedTest - @CsvSource({"blosclz,noshuffle,0", "lz4,shuffle,6", "lz4hc,bitshuffle,3", "zlib,shuffle,5", "zstd,bitshuffle,9"}) - public void testV2createBlosc(String cname, String shuffle, int clevel) throws IOException, ZarrException { - dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create( - new FilesystemStore(TESTOUTPUT).resolve("v2_create_blosc", cname + "_" + shuffle + "_" + clevel), - dev.zarr.zarrjava.v2.Array.metadataBuilder() - .withShape(10, 10) - .withDataType(dev.zarr.zarrjava.v2.DataType.UINT8) - .withChunks(5, 5) - .withFillValue(1) - .withBloscCompressor(cname, shuffle, clevel) - .build() - ); - array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); - - ucar.ma2.Array outArray = array.read(new long[]{2, 2}, new int[]{8, 8}); - Assertions.assertEquals(8 * 8, outArray.getSize()); - Assertions.assertEquals(0, outArray.getByte(0)); - } - - @Test - public void testV2create() throws IOException, ZarrException { - dev.zarr.zarrjava.v2.DataType dataType = dev.zarr.zarrjava.v2.DataType.UINT32; - - dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create( - new FilesystemStore(TESTOUTPUT).resolve("v2_create"), - dev.zarr.zarrjava.v2.Array.metadataBuilder() - .withShape(10, 10) - .withDataType(dataType) - .withChunks(5, 5) - .withFillValue(2) - .build() - ); - array.write(new long[]{2, 2}, ucar.ma2.Array.factory(dataType.getMA2DataType(), new int[]{8, 8})); - - ucar.ma2.Array outArray = array.read(new long[]{2, 2}, new int[]{8, 8}); - Assertions.assertEquals(8 * 8, outArray.getSize()); - Assertions.assertEquals(0, outArray.getByte(0)); - } - - @ParameterizedTest - @ValueSource(ints = {0, 1, 5, 9}) - public void testV2createZlib(int level) throws IOException, ZarrException { - dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create( - new FilesystemStore(TESTOUTPUT).resolve("v2_create_zlib", String.valueOf(level)), - dev.zarr.zarrjava.v2.Array.metadataBuilder() - .withShape(15, 10) - .withDataType(dev.zarr.zarrjava.v2.DataType.UINT8) - .withChunks(4, 5) - .withFillValue(5) - .withZlibCompressor(level) - .build() - ); - array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{7, 6})); - - ucar.ma2.Array outArray = array.read(new long[]{2, 2}, new int[]{7, 6}); - Assertions.assertEquals(7 * 6, outArray.getSize()); - Assertions.assertEquals(0, outArray.getByte(0)); - } - - @ParameterizedTest - @ValueSource(strings = {"BOOL", "INT8", "UINT8", "INT16", "UINT16", "INT32", "UINT32", "INT64", "UINT64", "FLOAT32", "FLOAT64"}) - public void testV2noFillValue(dev.zarr.zarrjava.v2.DataType dataType) throws IOException, ZarrException { - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("v2_no_fillvalue", dataType.name()); - - dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create( - storeHandle, - dev.zarr.zarrjava.v2.Array.metadataBuilder() - .withShape(15, 10) - .withDataType(dataType) - .withChunks(4, 5) - .build() - ); - Assertions.assertNull(array.metadata().fillValue); - - ucar.ma2.Array outArray = array.read(new long[]{0, 0}, new int[]{1, 1}); - if (dataType == dev.zarr.zarrjava.v2.DataType.BOOL) { - Assertions.assertFalse(outArray.getBoolean(0)); - } else { - Assertions.assertEquals(0, outArray.getByte(0)); - } - - dev.zarr.zarrjava.v2.Array array2 = dev.zarr.zarrjava.v2.Array.open( - storeHandle - ); - Assertions.assertNull(array2.metadata().fillValue); - } } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java new file mode 100644 index 0000000..765c51a --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java @@ -0,0 +1,258 @@ +package dev.zarr.zarrjava; + +import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.v2.Array; +import dev.zarr.zarrjava.v2.ArrayMetadata; +import dev.zarr.zarrjava.v2.DataType; +import dev.zarr.zarrjava.v2.Group; +import dev.zarr.zarrjava.v2.Node; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.Paths; + +public class ZarrV2Test extends ZarrTest { + @ParameterizedTest + @CsvSource({"blosclz,noshuffle,0", "lz4,shuffle,6", "lz4hc,bitshuffle,3", "zlib,shuffle,5", "zstd,bitshuffle,9"}) + public void testCreateBlosc(String cname, String shuffle, int clevel) throws IOException, ZarrException { + Array array = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("v2_create_blosc", cname + "_" + shuffle + "_" + clevel), + Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunks(5, 5) + .withFillValue(1) + .withBloscCompressor(cname, shuffle, clevel) + .build() + ); + array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); + + ucar.ma2.Array outArray = array.read(new long[]{2, 2}, new int[]{8, 8}); + Assertions.assertEquals(8 * 8, outArray.getSize()); + Assertions.assertEquals(0, outArray.getByte(0)); + } + + + + @ParameterizedTest + @CsvSource({ + "BOOL", "FLOAT64" + }) + public void testReadBloscDetectTypesize(DataType dt) throws IOException, ZarrException { + String arrayname = dt == DataType.BOOL ? "bool" : "double"; + StoreHandle storeHandle = new FilesystemStore(TESTDATA).resolve("v2_sample", arrayname); + Array array = Array.open(storeHandle); + ucar.ma2.Array output = array.read(new long[]{0, 0, 0}, new int[]{3, 4, 5}); + Assertions.assertEquals(dt, array.metadata().dataType); + } + + @Test + public void testCreate() throws IOException, ZarrException { + DataType dataType = DataType.UINT32; + + Array array = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("v2_create"), + Array.metadataBuilder() + .withShape(10, 10) + .withDataType(dataType) + .withChunks(5, 5) + .withFillValue(2) + .build() + ); + array.write(new long[]{2, 2}, ucar.ma2.Array.factory(dataType.getMA2DataType(), new int[]{8, 8})); + + ucar.ma2.Array outArray = array.read(new long[]{2, 2}, new int[]{8, 8}); + Assertions.assertEquals(8 * 8, outArray.getSize()); + Assertions.assertEquals(0, outArray.getByte(0)); + } + + @ParameterizedTest + @ValueSource(ints = {0, 1, 5, 9}) + public void testCreateZlib(int level) throws IOException, ZarrException { + Array array = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("v2_create_zlib", String.valueOf(level)), + Array.metadataBuilder() + .withShape(15, 10) + .withDataType(DataType.UINT8) + .withChunks(4, 5) + .withFillValue(5) + .withZlibCompressor(level) + .build() + ); + array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{7, 6})); + + ucar.ma2.Array outArray = array.read(new long[]{2, 2}, new int[]{7, 6}); + Assertions.assertEquals(7 * 6, outArray.getSize()); + Assertions.assertEquals(0, outArray.getByte(0)); + } + + @ParameterizedTest + @ValueSource(strings = {"BOOL", "INT8", "UINT8", "INT16", "UINT16", "INT32", "UINT32", "INT64", "UINT64", "FLOAT32", "FLOAT64"}) + public void testNoFillValue(DataType dataType) throws IOException, ZarrException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("v2_no_fillvalue", dataType.name()); + + Array array = Array.create( + storeHandle, + Array.metadataBuilder() + .withShape(15, 10) + .withDataType(dataType) + .withChunks(4, 5) + .build() + ); + Assertions.assertNull(array.metadata().fillValue); + + ucar.ma2.Array outArray = array.read(new long[]{0, 0}, new int[]{1, 1}); + if (dataType == DataType.BOOL) { + Assertions.assertFalse(outArray.getBoolean(0)); + } else { + Assertions.assertEquals(0, outArray.getByte(0)); + } + + Array array2 = Array.open( + storeHandle + ); + Assertions.assertNull(array2.metadata().fillValue); + } + + + @Test + public void testOpen() throws ZarrException, IOException { + StoreHandle arrayHandle = new FilesystemStore(TESTDATA).resolve("v2_sample", "subgroup", "array"); + StoreHandle groupHandle = new FilesystemStore(TESTDATA).resolve("v2_sample"); + StoreHandle v3Handle = new FilesystemStore(TESTDATA).resolve("l4_sample"); + + Array array = (Array) Node.open(arrayHandle); + Assertions.assertEquals(3, array.metadata().shape.length); + + array = (Array) dev.zarr.zarrjava.core.Array.open(arrayHandle); + Assertions.assertEquals(3, array.metadata().shape.length); + + array = (Array) dev.zarr.zarrjava.core.Node.open(arrayHandle); + Assertions.assertEquals(3, array.metadata().shape.length); + + Group group = (Group) Node.open(groupHandle); + Assertions.assertInstanceOf(Group.class, group.get("subgroup")); + + group = (Group) dev.zarr.zarrjava.core.Group.open(groupHandle); + Assertions.assertInstanceOf(Group.class, group.get("subgroup")); + + group = (Group) dev.zarr.zarrjava.core.Node.open(groupHandle); + Assertions.assertInstanceOf(Group.class, group.get("subgroup")); + + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(TESTDATA.resolve("non_existing"))); + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(v3Handle)); + Assertions.assertThrows(NoSuchFileException.class, () -> Group.open(v3Handle)); + Assertions.assertThrows(NoSuchFileException.class, () -> Array.open(v3Handle)); + } + + @Test + public void testOpenOverloads() throws ZarrException, IOException { + Path arrayPath = TESTDATA.resolve("v2_sample").resolve("subgroup").resolve("array"); + Path groupPath = TESTDATA.resolve("v2_sample"); + Path v3GroupPath = TESTDATA.resolve("l4_sample"); + + Array array = (Array) Node.open(arrayPath); + Assertions.assertEquals(3, array.metadata().shape.length); + array = (Array) Node.open(arrayPath.toString()); + Assertions.assertEquals(3, array.metadata().shape.length); + + array = (Array) dev.zarr.zarrjava.core.Array.open(arrayPath); + Assertions.assertEquals(3, array.metadata().shape.length); + array = (Array) dev.zarr.zarrjava.core.Array.open(arrayPath.toString()); + Assertions.assertEquals(3, array.metadata().shape.length); + + array = (Array) dev.zarr.zarrjava.core.Node.open(arrayPath); + Assertions.assertEquals(3, array.metadata().shape.length); + array = (Array) dev.zarr.zarrjava.core.Node.open(arrayPath.toString()); + Assertions.assertEquals(3, array.metadata().shape.length); + + Group group = (Group) Node.open(groupPath); + Assertions.assertInstanceOf(Group.class, group.get("subgroup")); + group = (Group) Node.open(groupPath.toString()); + Assertions.assertInstanceOf(Group.class, group.get("subgroup")); + + group = (Group) dev.zarr.zarrjava.core.Group.open(groupPath); + Assertions.assertInstanceOf(Group.class, group.get("subgroup")); + group = (Group) dev.zarr.zarrjava.core.Group.open(groupPath.toString()); + Assertions.assertInstanceOf(Group.class, group.get("subgroup")); + + group = (Group) dev.zarr.zarrjava.core.Node.open(groupPath); + Assertions.assertInstanceOf(Group.class, group.get("subgroup")); + group = (Group) dev.zarr.zarrjava.core.Node.open(groupPath.toString()); + Assertions.assertInstanceOf(Group.class, group.get("subgroup")); + + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(TESTDATA.resolve("non_existing"))); + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(TESTDATA.resolve("non_existing").toString())); + + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(v3GroupPath)); + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(v3GroupPath.toString())); + + Assertions.assertThrows(NoSuchFileException.class, () -> Group.open(v3GroupPath)); + Assertions.assertThrows(NoSuchFileException.class, () -> Group.open(v3GroupPath.toString())); + + Assertions.assertThrows(NoSuchFileException.class, () -> Array.open(v3GroupPath)); + Assertions.assertThrows(NoSuchFileException.class, () -> Array.open(v3GroupPath.toString())); + } + + @Test + public void testCreateArray() throws ZarrException, IOException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCreateArrayV2"); + Path storeHandlePath = TESTOUTPUT.resolve("testCreateArrayV2Path"); + String storeHandleString = String.valueOf(TESTOUTPUT.resolve("testCreateArrayV2String")); + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunks(5, 5) + .build(); + + Array.create(storeHandle, arrayMetadata); + Assertions.assertTrue(storeHandle.resolve(".zarray").exists()); + + Array.create(storeHandlePath, arrayMetadata); + Assertions.assertTrue(Files.exists(storeHandlePath.resolve(".zarray"))); + + Array.create(storeHandleString, arrayMetadata); + Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve(".zarray"))); + } + + @Test + public void testGroup() throws IOException, ZarrException { + FilesystemStore fsStore = new FilesystemStore(TESTOUTPUT); + + Group group = Group.create(fsStore.resolve("v2_testgroup")); + Group group2 = group.createGroup("test2"); + Array array = group2.createArray("array", b -> + b.withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunks(5, 5) + ); + array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); + + Assertions.assertArrayEquals(new int[]{5, 5}, ((Array) ((Group) group.listAsArray()[0]).listAsArray()[0]).metadata().chunks); + } + + + @Test + public void testCreateGroup() throws ZarrException, IOException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCreateGroupV2"); + Path storeHandlePath = TESTOUTPUT.resolve("testCreateGroupV2Path"); + String storeHandleString = String.valueOf(TESTOUTPUT.resolve("testCreateGroupV2String")); + + Group.create(storeHandle); + Assertions.assertTrue(storeHandle.resolve(".zgroup").exists()); + + Group.create(storeHandlePath); + Assertions.assertTrue(Files.exists(storeHandlePath.resolve(".zgroup"))); + + Group.create(storeHandleString); + Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve(".zgroup"))); + } +} \ No newline at end of file diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java new file mode 100644 index 0000000..7f1c15c --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java @@ -0,0 +1,574 @@ +package dev.zarr.zarrjava; + +import dev.zarr.zarrjava.v3.codec.core.BloscCodec; +import dev.zarr.zarrjava.v3.codec.core.ShardingIndexedCodec; + +import com.fasterxml.jackson.databind.JsonMappingException; +import dev.zarr.zarrjava.store.*; +import dev.zarr.zarrjava.utils.MultiArrayUtils; +import dev.zarr.zarrjava.v3.Node; +import dev.zarr.zarrjava.v3.*; +import dev.zarr.zarrjava.v3.codec.CodecBuilder; +import dev.zarr.zarrjava.v3.codec.core.BytesCodec; +import dev.zarr.zarrjava.v3.codec.core.TransposeCodec; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import ucar.ma2.MAMath; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Stream; + +import static dev.zarr.zarrjava.core.ArrayMetadata.parseFillValue; +import static org.junit.Assert.assertThrows; + +public class ZarrV3Test extends ZarrTest { + + static Stream> invalidCodecBuilder() { + return Stream.of( + c -> c.withBytes(BytesCodec.Endian.LITTLE).withBytes(BytesCodec.Endian.LITTLE), + c -> c.withBlosc().withBytes(BytesCodec.Endian.LITTLE), + c -> c.withBytes(BytesCodec.Endian.LITTLE).withTranspose(new int[]{1, 0}), + c -> c.withTranspose(new int[]{1, 0}).withBytes(BytesCodec.Endian.LITTLE).withTranspose(new int[]{1, 0}) + ); + } + + @ParameterizedTest + @MethodSource("invalidCodecBuilder") + public void testCheckInvalidCodecConfiguration(Function codecBuilder) { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("invalid_codec_config", String.valueOf(codecBuilder.hashCode())); + ArrayMetadataBuilder builder = Array.metadataBuilder() + .withShape(4, 4) + .withDataType(DataType.UINT32) + .withChunkShape(2, 2) + .withCodecs(codecBuilder); + + assertThrows(ZarrException.class, () -> Array.create(storeHandle, builder.build())); + } + + @Test + public void testLargerChunkSizeThanArraySize() throws ZarrException, IOException { + int[] testData = new int[16 * 16 * 16]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("larger_chunk_size_than_array"); + ArrayMetadata metadata = Array.metadataBuilder() + .withShape(16, 16, 16) + .withDataType(DataType.UINT32) + .withChunkShape(32, 32, 32) + .withFillValue(0) + .build(); + Array writeArray = Array.create(storeHandle, metadata); + writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); + + //read in zarr-java + Array readArray = Array.open(storeHandle); + ucar.ma2.Array result = readArray.read(); + + Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + } + + static Stream invalidChunkSizes() { + return Stream.of( + new int[]{1}, + new int[]{1, 1, 1} + ); + } + + @ParameterizedTest + @MethodSource("invalidChunkSizes") + public void testCheckInvalidChunkDimensions(int[] chunkSize) { + long[] shape = new long[]{4, 4}; + + ArrayMetadataBuilder builder = Array.metadataBuilder() + .withShape(shape) + .withDataType(DataType.UINT32) + .withChunkShape(chunkSize); + + assertThrows(ZarrException.class, builder::build); + } + + static Stream invalidShardSizes() { + return Stream.of( + new int[]{4}, //wrong dims + new int[]{4, 4, 4}, //wrong dims + new int[]{1, 1}, //smaller than inner chunk shape + new int[]{5, 5}, //no exact multiple of inner chunk shape + new int[]{2, 1}, //smaller than inner chunk shape in 2nd dimension + new int[]{2, 5} //no exact multiple of inner chunk shape in 2nd dimension + ); + } + static Stream invalidShardSizesWithNested() { + return invalidShardSizes().flatMap(shardSize -> + Stream.of(true, false).map(nested -> Arguments.of(shardSize, nested)) + ); + } + @ParameterizedTest + @MethodSource("invalidShardSizesWithNested") + public void testCheckShardingBounds(int[] shardSize, boolean nested) { + long[] shape = new long[]{10, 10}; + int[] innerChunkSize = new int[]{2, 2}; + + ArrayMetadataBuilder builder = Array.metadataBuilder() + .withShape(shape) + .withDataType(DataType.UINT32).withChunkShape(shardSize); + + if (nested) { + int[] nestedChunkSize = new int[]{4, 4}; + builder = builder.withCodecs(c -> c.withSharding(new int[]{2, 2}, c1 -> c1.withSharding(nestedChunkSize, c2 -> c2.withBytes("LITTLE")))); + } + builder = builder.withCodecs(c -> c.withSharding(innerChunkSize, c1 -> c1.withBytes("LITTLE"))); + assertThrows(ZarrException.class, builder::build); + } + + @ParameterizedTest + @CsvSource({"0,true", "0,false", "5, true", "5, false"}) + public void testZstdCodecReadWrite(int clevel, boolean checksum) throws ZarrException, IOException { + int[] testData = new int[16 * 16 * 16]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testZstdCodecReadWrite", "checksum_" + checksum, "clevel_" + clevel); + ArrayMetadataBuilder builder = Array.metadataBuilder() + .withShape(16, 16, 16) + .withDataType(DataType.UINT32) + .withChunkShape(2, 4, 8) + .withFillValue(0) + .withCodecs(c -> c.withZstd(clevel, checksum)); + Array writeArray = Array.create(storeHandle, builder.build()); + writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); + + Array readArray = Array.open(storeHandle); + ucar.ma2.Array result = readArray.read(); + + Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + } + + @Test + public void testTransposeCodec() throws ZarrException { + ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{2, 3, 3}, new int[]{ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}); + ucar.ma2.Array testDataTransposed120 = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{3, 3, 2}, new int[]{ + 0, 9, 1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7, 16, 8, 17}); + + TransposeCodec transposeCodec = new TransposeCodec(new TransposeCodec.Configuration(new int[]{1, 2, 0})); + transposeCodec.setCoreArrayMetadata(new ArrayMetadata.CoreArrayMetadata( + new long[]{2, 3, 3}, + new int[]{2, 3, 3}, + DataType.UINT32, + null)); + + assert MAMath.equals(testDataTransposed120, transposeCodec.encode(testData)); + assert MAMath.equals(testData, transposeCodec.decode(testDataTransposed120)); + } + + static Stream invalidTransposeOrder() { + return Stream.of( + new int[]{1, 0, 0}, + new int[]{1, 2, 3}, + new int[]{1, 2, 3, 0}, + new int[]{1, 2} + ); + } + + @ParameterizedTest + @MethodSource("invalidTransposeOrder") + public void testCheckInvalidTransposeOrder(int[] transposeOrder) throws Exception { + int[] shapeInt = new int[]{2, 3, 3}; + long[] shapeLong = new long[]{2, 3, 3}; + + TransposeCodec transposeCodec = new TransposeCodec(new TransposeCodec.Configuration(transposeOrder)); + transposeCodec.setCoreArrayMetadata(new ArrayMetadata.CoreArrayMetadata( + shapeLong, + shapeInt, + DataType.UINT32, + null)); + + ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, shapeInt); + assertThrows(ZarrException.class, () -> transposeCodec.encode(testData)); + } + + @Test + public void testShardingReadCutout() throws IOException, ZarrException { + Array array = Array.open(new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1")); + + ucar.ma2.Array outArray = array.read(new long[]{0, 3073, 3073, 513}, new int[]{1, 64, 64, 64}); + Assertions.assertEquals(64 * 64 * 64, outArray.getSize()); + Assertions.assertEquals(-98, outArray.getByte(0)); + } + + @Test + public void testAccess() throws IOException, ZarrException { + Array readArray = Array.open(new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1")); + + ucar.ma2.Array outArray = readArray.access().withOffset(0, 3073, 3073, 513) + .withShape(1, 64, 64, 64) + .read(); + Assertions.assertEquals(64 * 64 * 64, outArray.getSize()); + Assertions.assertEquals(-98, outArray.getByte(0)); + + Array writeArray = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("l4_sample_2", "color", "1"), + readArray.metadata() + ); + writeArray.access().withOffset(0, 3073, 3073, 513).write(outArray); + } + + @ParameterizedTest + @ValueSource(strings = {"start", "end"}) + public void testShardingReadWrite(String indexLocation) throws IOException, ZarrException { + Array readArray = Array.open( + new FilesystemStore(TESTDATA).resolve("sharding_index_location", indexLocation)); + ucar.ma2.Array readArrayContent = readArray.read(); + Array writeArray = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("sharding_index_location", indexLocation), + readArray.metadata() + ); + writeArray.write(readArrayContent); + ucar.ma2.Array outArray = writeArray.read(); + + assert MultiArrayUtils.allValuesEqual(readArrayContent, outArray); + } + + @Test + public void testCodecs() throws IOException, ZarrException { + int[] readShape = new int[]{1, 1, 1024, 1024}; + Array readArray = Array.open( + new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "8-8-2")); + ucar.ma2.Array readArrayContent = readArray.read(new long[4], readShape); + { + Array gzipArray = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("l4_sample_gzip", "color", "8-8-2"), + Array.metadataBuilder(readArray.metadata()).withCodecs(c -> c.withGzip(5)).build() + ); + gzipArray.write(readArrayContent); + ucar.ma2.Array outGzipArray = gzipArray.read(new long[4], readShape); + assert MultiArrayUtils.allValuesEqual(outGzipArray, readArrayContent); + } + { + Array bloscArray = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("l4_sample_blosc", "color", "8-8-2"), + Array.metadataBuilder(readArray.metadata()).withCodecs(c -> c.withBlosc("zstd", 5)).build() + ); + bloscArray.write(readArrayContent); + ucar.ma2.Array outBloscArray = bloscArray.read(new long[4], readShape); + assert MultiArrayUtils.allValuesEqual(outBloscArray, readArrayContent); + } + { + Array zstdArray = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("l4_sample_zstd", "color", "8-8-2"), + Array.metadataBuilder(readArray.metadata()).withCodecs(c -> c.withZstd(10)).build() + ); + zstdArray.write(readArrayContent); + ucar.ma2.Array outZstdArray = zstdArray.read(new long[4], readShape); + assert MultiArrayUtils.allValuesEqual(outZstdArray, readArrayContent); + } + } + + @Test + public void testArrayMetadataBuilder() throws ZarrException { + long[] shape = new long[]{1, 4096, 4096, 1536}; + DataType dataType = DataType.UINT32; + int[] chunkShape = new int[]{1, 1024, 1024, 1024}; + int fillValue = 0; + + ArrayMetadata metadata = Array.metadataBuilder() + .withShape(shape) + .withDataType(dataType) + .withChunkShape(chunkShape) + .withFillValue(fillValue) + .withCodecs( + c -> c.withSharding(new int[]{1, 32, 32, 32}, CodecBuilder::withBlosc)) + .build(); + Assertions.assertArrayEquals(shape, metadata.shape); + Assertions.assertEquals(dataType, metadata.dataType); + Assertions.assertArrayEquals(chunkShape, metadata.chunkShape()); + Assertions.assertEquals(fillValue, metadata.fillValue); + Assertions.assertEquals(1, metadata.codecs.length); + ShardingIndexedCodec shardingCodec = (ShardingIndexedCodec) metadata.codecs[0]; + Assertions.assertInstanceOf(ShardingIndexedCodec.class, shardingCodec); + Assertions.assertInstanceOf(BytesCodec.class, shardingCodec.configuration.codecs[0]); + Assertions.assertInstanceOf(BloscCodec.class, shardingCodec.configuration.codecs[1]); + } + + @Test + public void testFillValue() throws ZarrException { + Assertions.assertEquals(0, (int) parseFillValue(0, DataType.UINT32)); + Assertions.assertEquals(50462976, (int) parseFillValue("0x00010203", DataType.UINT32)); + Assertions.assertEquals(2, (byte) parseFillValue("0b00000010", DataType.UINT8)); + assert Double.isNaN((double) parseFillValue("NaN", DataType.FLOAT64)); + assert Double.isInfinite((double) parseFillValue("-Infinity", DataType.FLOAT64)); + } + + @Test + public void testReadme1() throws IOException, ZarrException { + Group hierarchy = Group.open( + new HttpStore("https://static.webknossos.org/data/zarr_v3") + .resolve("l4_sample") + ); + Group color = (Group) hierarchy.get("color"); + Array array = (Array) color.get("1"); + ucar.ma2.Array outArray = array.read( + new long[]{0, 3073, 3073, 513}, // offset + new int[]{1, 64, 64, 64} // shape + ); + Assertions.assertEquals(64 * 64 * 64, outArray.getSize()); + } + + @Test + public void testReadme2() throws IOException, ZarrException { + Array array = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("testoutput", "color", "1"), + Array.metadataBuilder() + .withShape(1, 4096, 4096, 1536) + .withDataType(DataType.UINT32) + .withChunkShape(1, 1024, 1024, 1024) + .withFillValue(0) + .withCodecs(c -> c.withSharding(new int[]{1, 32, 32, 32}, c1 -> c1.withBlosc())) + .build() + ); + ucar.ma2.Array data = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1, 1, 2, 2}, new int[]{1, 2, 3, 4}); + array.write( + new long[]{0, 0, 0, 0}, // offset + data + ); + ucar.ma2.Array output = array.read(new long[]{0, 0, 0, 0}, new int[]{1, 1, 2, 2}); + assert MultiArrayUtils.allValuesEqual(data, output); + } + + @ParameterizedTest + @ValueSource(strings = {"1", "2-2-1", "4-4-1", "16-16-4"}) + public void testReadL4Sample(String mag) throws IOException, ZarrException { + StoreHandle httpStoreHandle = new HttpStore("https://static.webknossos.org/data/zarr_v3/").resolve("l4_sample", "color", mag); + StoreHandle localStoreHandle = new FilesystemStore(TESTDATA).resolve("l4_sample", "color", mag); + + Array httpArray = Array.open(httpStoreHandle); + Array localArray = Array.open(localStoreHandle); + + Assertions.assertArrayEquals(httpArray.metadata().shape, localArray.metadata().shape); + Assertions.assertArrayEquals(httpArray.metadata().chunkShape(), localArray.metadata().chunkShape()); + + ucar.ma2.Array httpData1 = httpArray.read(new long[]{0, 0, 0, 0}, new int[]{1, 64, 64, 64}); + ucar.ma2.Array localData1 = localArray.read(new long[]{0, 0, 0, 0}, new int[]{1, 64, 64, 64}); + + assert MultiArrayUtils.allValuesEqual(httpData1, localData1); + + //offset to where l4_sample contains non-zero values + long[] offset = new long[4]; + long[] originalOffset = new long[]{0, 3073, 3073, 513}; + long[] originalShape = new long[]{1, 4096, 4096, 2048}; + long[] arrayShape = httpArray.metadata().shape; + for (int i = 0; i < 4; i++) { + offset[i] = originalOffset[i] / (originalShape[i] / arrayShape[i]); + } + + ucar.ma2.Array httpData2 = httpArray.read(offset, new int[]{1, 64, 64, 64}); + ucar.ma2.Array localData2 = localArray.read(offset, new int[]{1, 64, 64, 64}); + + assert MultiArrayUtils.allValuesEqual(httpData2, localData2); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testParallel(boolean useParallel) throws IOException, ZarrException { + int[] testData = new int[512 * 512 * 512]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testParallelRead"); + ArrayMetadata metadata = Array.metadataBuilder() + .withShape(512, 512, 512) + .withDataType(DataType.UINT32) + .withChunkShape(100, 100, 100) + .withFillValue(0) + .build(); + Array writeArray = Array.create(storeHandle, metadata); + writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{512, 512, 512}, testData), useParallel); + + Array readArray = Array.open(storeHandle); + ucar.ma2.Array result = readArray.read(useParallel); + + Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + clearTestoutputFolder(); + } + + @Test + public void testMetadataAcceptsEmptyStorageTransformer() throws ZarrException, IOException { + // non-empty storage transformers are currently not supported + + Map[] storageTransformersEmpty = Array.open( + new FilesystemStore(TESTDATA).resolve("storage_transformer", "empty") + ).metadata().storageTransformers; + assert storageTransformersEmpty.length == 0; + + assertThrows(JsonMappingException.class, () -> Array.open( + new FilesystemStore(TESTDATA).resolve("storage_transformer", "exists")) + ); + + ArrayMetadataBuilder builderWithStorageTransformer = Array.metadataBuilder() + .withShape(1) + .withChunkShape(1) + .withDataType(DataType.UINT8) + .withStorageTransformers(new HashMap[]{new HashMap() {{ + put("some", "value"); + }}}); + + assertThrows(ZarrException.class, () -> Array.create( + new FilesystemStore(TESTOUTPUT).resolve("storage_transformer"), + builderWithStorageTransformer.build() + )); + } + + @Test + public void testOpen() throws ZarrException, IOException { + StoreHandle arrayHandle = new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1"); + StoreHandle groupHandle = new FilesystemStore(TESTDATA).resolve("l4_sample"); + StoreHandle v2Handle = new FilesystemStore(TESTDATA).resolve("v2_sample"); + + Array array = (Array) Node.open(arrayHandle); + Assertions.assertEquals(4, array.metadata().shape.length); + + array = (Array) dev.zarr.zarrjava.core.Array.open(arrayHandle); + Assertions.assertEquals(4, array.metadata().shape.length); + + array = (Array) dev.zarr.zarrjava.core.Node.open(arrayHandle); + Assertions.assertEquals(4, array.metadata().shape.length); + + Group group = (Group) Node.open(groupHandle); + Assertions.assertInstanceOf(Group.class, group.get("color")); + + group = (Group) dev.zarr.zarrjava.core.Group.open(groupHandle); + Assertions.assertInstanceOf(Group.class, group.get("color")); + + group = (Group) dev.zarr.zarrjava.core.Node.open(groupHandle); + Assertions.assertInstanceOf(Group.class, group.get("color")); + + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(TESTDATA.resolve("non_existing"))); + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(v2Handle)); + Assertions.assertThrows(NoSuchFileException.class, () -> Group.open(v2Handle)); + Assertions.assertThrows(NoSuchFileException.class, () -> Array.open(v2Handle)); + } + + @Test + public void testOpenOverloads() throws ZarrException, IOException { + Path arrayPath = TESTDATA.resolve("l4_sample").resolve("color").resolve("1"); + Path groupPath = TESTDATA.resolve("l4_sample"); + Path v2GroupPath = TESTDATA.resolve("v2_sample"); + + Array array = (Array) Node.open(arrayPath); + Assertions.assertEquals(4, array.metadata().shape.length); + array = (Array) Node.open(arrayPath.toString()); + Assertions.assertEquals(4, array.metadata().shape.length); + + array = (Array) dev.zarr.zarrjava.core.Array.open(arrayPath); + Assertions.assertEquals(4, array.metadata().shape.length); + array = (Array) dev.zarr.zarrjava.core.Array.open(arrayPath.toString()); + Assertions.assertEquals(4, array.metadata().shape.length); + + array = (Array) dev.zarr.zarrjava.core.Node.open(arrayPath); + Assertions.assertEquals(4, array.metadata().shape.length); + array = (Array) dev.zarr.zarrjava.core.Node.open(arrayPath.toString()); + Assertions.assertEquals(4, array.metadata().shape.length); + + Group group = (Group) Node.open(groupPath); + Assertions.assertInstanceOf(Group.class, group.get("color")); + group = (Group) Node.open(groupPath.toString()); + Assertions.assertInstanceOf(Group.class, group.get("color")); + + group = (Group) dev.zarr.zarrjava.core.Group.open(groupPath); + Assertions.assertInstanceOf(Group.class, group.get("color")); + group = (Group) dev.zarr.zarrjava.core.Group.open(groupPath.toString()); + Assertions.assertInstanceOf(Group.class, group.get("color")); + + group = (Group) dev.zarr.zarrjava.core.Node.open(groupPath); + Assertions.assertInstanceOf(Group.class, group.get("color")); + group = (Group) dev.zarr.zarrjava.core.Node.open(groupPath.toString()); + Assertions.assertInstanceOf(Group.class, group.get("color")); + + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(TESTDATA.resolve("non_existing"))); + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(TESTDATA.resolve("non_existing").toString())); + + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(v2GroupPath)); + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(v2GroupPath.toString())); + + Assertions.assertThrows(NoSuchFileException.class, () -> Group.open(v2GroupPath)); + Assertions.assertThrows(NoSuchFileException.class, () -> Group.open(v2GroupPath.toString())); + + Assertions.assertThrows(NoSuchFileException.class, () -> Array.open(v2GroupPath)); + Assertions.assertThrows(NoSuchFileException.class, () -> Array.open(v2GroupPath.toString())); + } + + @Test + public void testGroup() throws IOException, ZarrException { + FilesystemStore fsStore = new FilesystemStore(TESTOUTPUT); + + Map attributes = new HashMap<>(); + attributes.put("hello", "world"); + + Group group = Group.create(fsStore.resolve("testgroup")); + Group group2 = group.createGroup("test2", attributes); + Array array = group2.createArray("array", b -> + b.withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunkShape(5, 5) + ); + array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); + + Assertions.assertArrayEquals(new int[]{5, 5}, ((Array) ((Group) group.listAsArray()[0]).listAsArray()[0]).metadata().chunkShape()); + } + + @Test + public void testCreateArray() throws ZarrException, IOException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCreateArrayV3"); + Path storeHandlePath = TESTOUTPUT.resolve("testCreateArrayV3Path"); + String storeHandleString = String.valueOf(TESTOUTPUT.resolve("testCreateArrayV3String")); + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunkShape(5, 5) + .build(); + + Array.create(storeHandle, arrayMetadata); + Assertions.assertTrue(storeHandle.resolve("zarr.json").exists()); + + Array.create(storeHandlePath, arrayMetadata); + Assertions.assertTrue(Files.exists(storeHandlePath.resolve("zarr.json"))); + + Array.create(storeHandleString, arrayMetadata); + Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve("zarr.json"))); + } + + @Test + public void testCreateGroup() throws ZarrException, IOException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCreateGroupV3"); + Path storeHandlePath = TESTOUTPUT.resolve("testCreateGroupV3Path"); + String storeHandleString = String.valueOf(TESTOUTPUT.resolve("testCreateGroupV3String")); + Map attributes = new HashMap<>(); + attributes.put("hello", "world"); + + Group group = Group.create(storeHandle, new GroupMetadata(attributes)); + Assertions.assertTrue(storeHandle.resolve("zarr.json").exists()); + Assertions.assertEquals("world", group.metadata.attributes.get("hello")); + + group = Group.create(storeHandlePath, new GroupMetadata(attributes)); + Assertions.assertTrue(Files.exists(storeHandlePath.resolve("zarr.json"))); + Assertions.assertEquals("world", group.metadata.attributes.get("hello")); + + group = Group.create(storeHandleString, new GroupMetadata(attributes)); + Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve("zarr.json"))); + Assertions.assertEquals("world", group.metadata.attributes.get("hello")); + } + + + +} diff --git a/src/test/python-scripts/parse_codecs.py b/src/test/python-scripts/parse_codecs.py index d720706..772b34d 100644 --- a/src/test/python-scripts/parse_codecs.py +++ b/src/test/python-scripts/parse_codecs.py @@ -6,7 +6,6 @@ from zarr.codecs.sharding import ShardingCodec, ShardingCodecIndexLocation from zarr.codecs.transpose import TransposeCodec from zarr.codecs.zstd import ZstdCodec -import zarrita import numcodecs def parse_codecs_zarr_python(codec_string: str, param_string: str, zarr_version: int = 3): @@ -16,7 +15,7 @@ def parse_codecs_zarr_python(codec_string: str, param_string: str, zarr_version: if codec_string == "blosc" and zarr_version == 3: cname, shuffle, clevel = param_string.split("_") - compressor = BloscCodec(typesize=4, cname=cname, shuffle=shuffle, clevel=int(clevel)) + compressor = BloscCodec(cname=cname, shuffle=shuffle, clevel=int(clevel)) elif codec_string == "blosc" and zarr_version == 2: cname, shuffle, clevel = param_string.split("_") if shuffle == "noshuffle": @@ -51,33 +50,4 @@ def parse_codecs_zarr_python(codec_string: str, param_string: str, zarr_version: else: raise ValueError(f"Invalid codec: {codec_string}, zarr_version: {zarr_version}") - return compressor, serializer, filters - -def parse_codecs_zarrita(codec_string: str, param_string: str): - codec = [] - if codec_string == "blosc": - cname, shuffle, clevel = param_string.split("_") - codec = [zarrita.codecs.bytes_codec(), - zarrita.codecs.blosc_codec(typesize=4, cname=cname, shuffle=shuffle, clevel=int(clevel))] - elif codec_string == "gzip": - codec = [zarrita.codecs.bytes_codec(), zarrita.codecs.gzip_codec(level=int(param_string))] - elif codec_string == "zstd": - level, checksum = param_string.split("_") - codec = [zarrita.codecs.bytes_codec(), zarrita.codecs.zstd_codec(checksum=checksum == 'true', level=int(level))] - elif codec_string == "bytes": - codec = [zarrita.codecs.bytes_codec(endian=param_string.lower())] - elif codec_string == "transpose": - codec = [zarrita.codecs.transpose_codec((1, 0, 2)), zarrita.codecs.bytes_codec()] - elif codec_string == "sharding": - codec = zarrita.codecs.sharding_codec(chunk_shape=(2, 2, 4), codecs=[zarrita.codecs.bytes_codec("little")], - index_location=zarrita.metadata.ShardingCodecIndexLocation.start if param_string == "start" - else zarrita.metadata.ShardingCodecIndexLocation.end), - elif codec_string == "sharding_nested": - codec = zarrita.codecs.sharding_codec(chunk_shape=(2, 2, 4), - codecs=[zarrita.codecs.sharding_codec(chunk_shape=(2, 1, 2), codecs=[ - zarrita.codecs.bytes_codec("little")])]), - elif codec_string == "crc32c": - codec = [zarrita.codecs.bytes_codec(), zarrita.codecs.crc32c_codec()] - else: - raise ValueError(f"Invalid codec: {codec_string}") - return codec + return compressor, serializer, filters \ No newline at end of file diff --git a/src/test/python-scripts/zarr_python_group_v2.py b/src/test/python-scripts/zarr_python_group_v2.py new file mode 100644 index 0000000..5052ddf --- /dev/null +++ b/src/test/python-scripts/zarr_python_group_v2.py @@ -0,0 +1,27 @@ +import sys +from pathlib import Path + +import numpy as np + +import zarr +from zarr.storage import LocalStore + +store_path_read = Path(sys.argv[1]) + +expected_data = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) + +g = zarr.open_group(store=LocalStore(store_path_read), zarr_format=2) +a = g['group']['array'] +read_data = a[:, :] +assert np.array_equal(read_data, expected_data), f"got:\n {read_data} \nbut expected:\n {expected_data}" + +store_path_write = Path(sys.argv[2]) +g2 = zarr.create_group(store=LocalStore(store_path_write), zarr_format=2) +a2 = g2.create_group('group2').create_array( + name='array2', + shape=(16, 16, 16), + chunks=(2, 4, 8), + dtype="int32", + fill_value=0, +) +a2[:] = expected_data \ No newline at end of file diff --git a/src/test/python-scripts/zarr_python_read.py b/src/test/python-scripts/zarr_python_read.py index 1e31ff5..d6cee0c 100644 --- a/src/test/python-scripts/zarr_python_read.py +++ b/src/test/python-scripts/zarr_python_read.py @@ -10,9 +10,13 @@ codec_string = sys.argv[1] param_string = sys.argv[2] compressor, serializer, filters = parse_codecs_zarr_python(codec_string, param_string) -store_path = Path(sys.argv[3]) +dtype = sys.argv[3] +store_path = Path(sys.argv[4]) -expected_data = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) +if dtype == 'bool': + expected_data = np.arange(16 * 16 * 16, dtype='uint8').reshape(16, 16, 16) % 2 == 0 +else: + expected_data = np.arange(16 * 16 * 16, dtype=dtype).reshape(16, 16, 16) a = zarr.open_array(store=LocalStore(store_path)) read_data = a[:, :] @@ -22,7 +26,7 @@ LocalStore(store_path / "expected"), shape=(16, 16, 16), chunks=(2, 4, 8), - dtype="uint32", + dtype=dtype, fill_value=0, filters=filters, serializer=serializer, diff --git a/src/test/python-scripts/zarr_python_read_v2.py b/src/test/python-scripts/zarr_python_read_v2.py index 5390a23..5ff5e5a 100644 --- a/src/test/python-scripts/zarr_python_read_v2.py +++ b/src/test/python-scripts/zarr_python_read_v2.py @@ -11,9 +11,13 @@ codec_string = sys.argv[1] param_string = sys.argv[2] compressor, serializer, filters = parse_codecs_zarr_python(codec_string, param_string, zarr_version=2) -store_path = Path(sys.argv[3]) +dtype = sys.argv[3] +store_path = Path(sys.argv[4]) -expected_data = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) +if dtype == 'bool': + expected_data = np.arange(16 * 16 * 16, dtype='uint8').reshape(16, 16, 16) % 2 == 0 +else: + expected_data = np.arange(16 * 16 * 16, dtype=dtype).reshape(16, 16, 16) a = zarr.open_array(store=LocalStore(store_path)) read_data = a[:, :] @@ -24,7 +28,7 @@ zarr_format=2, shape=(16, 16, 16), chunks=(2, 4, 8), - dtype='uint32', + dtype=dtype, fill_value=0, filters=filters, serializer=serializer, diff --git a/src/test/python-scripts/zarr_python_write.py b/src/test/python-scripts/zarr_python_write.py index 531d356..328400f 100644 --- a/src/test/python-scripts/zarr_python_write.py +++ b/src/test/python-scripts/zarr_python_write.py @@ -10,15 +10,20 @@ codec_string = sys.argv[1] param_string = sys.argv[2] compressor, serializer, filters = parse_codecs_zarr_python(codec_string, param_string) -store_path = Path(sys.argv[3]) +dtype = sys.argv[3] +store_path = Path(sys.argv[4]) + +if dtype == 'bool': + testdata = np.arange(16 * 16 * 16, dtype='uint8').reshape(16, 16, 16) % 2 == 0 +else: + testdata = np.arange(16 * 16 * 16, dtype=dtype).reshape(16, 16, 16) -testdata = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) a = zarr.create_array( LocalStore(store_path), shape=(16, 16, 16), chunks=(2, 4, 8), - dtype='int32', + dtype=dtype, filters=filters, serializer=serializer, compressors=compressor, diff --git a/src/test/python-scripts/zarr_python_write_v2.py b/src/test/python-scripts/zarr_python_write_v2.py index 9c124c1..0068e04 100644 --- a/src/test/python-scripts/zarr_python_write_v2.py +++ b/src/test/python-scripts/zarr_python_write_v2.py @@ -11,16 +11,20 @@ codec_string = sys.argv[1] param_string = sys.argv[2] compressor, serializer, filters = parse_codecs_zarr_python(codec_string, param_string, zarr_version=2) -store_path = Path(sys.argv[3]) +dtype = sys.argv[3] +store_path = Path(sys.argv[4]) -testdata = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) +if dtype == 'bool': + testdata = np.arange(16 * 16 * 16, dtype='uint8').reshape(16, 16, 16) % 2 == 0 +else: + testdata = np.arange(16 * 16 * 16, dtype=dtype).reshape(16, 16, 16) a = zarr.create_array( LocalStore(store_path), zarr_format=2, shape=(16, 16, 16), chunks=(2, 4, 8), - dtype='int32', + dtype=dtype, filters=filters, serializer=serializer, compressors=compressor, diff --git a/src/test/python-scripts/zarrita_read.py b/src/test/python-scripts/zarrita_read.py deleted file mode 100644 index 2769694..0000000 --- a/src/test/python-scripts/zarrita_read.py +++ /dev/null @@ -1,29 +0,0 @@ -import sys - -import numpy as np -import zarrita -from zarrita.metadata import ShardingCodecIndexLocation -from parse_codecs import parse_codecs_zarrita - -codec_string = sys.argv[1] -param_string = sys.argv[2] -codec = parse_codecs_zarrita(codec_string, param_string) - -store = zarrita.LocalStore(sys.argv[3]) -expected_data = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) - -a = zarrita.Array.open(store / 'write_to_zarrita' / codec_string / param_string) -read_data = a[:, :] -assert np.array_equal(read_data, expected_data), f"got:\n {read_data} \nbut expected:\n {expected_data}" - -b = zarrita.Array.create( - store / 'read_from_zarrita_expected' / codec_string / param_string, - shape=(16, 16, 16), - chunk_shape=(2, 4, 8), - dtype="uint32", - fill_value=0, - attributes={'test_key': 'test_value'}, - codecs=codec -) - -assert a.metadata == b.metadata, f"not equal: \n{a.metadata=}\n{b.metadata=}" diff --git a/src/test/python-scripts/zarrita_write.py b/src/test/python-scripts/zarrita_write.py deleted file mode 100644 index 4000f60..0000000 --- a/src/test/python-scripts/zarrita_write.py +++ /dev/null @@ -1,23 +0,0 @@ -import sys - -import zarrita -import numpy as np -from zarrita.metadata import ShardingCodecIndexLocation -from parse_codecs import parse_codecs_zarrita - -codec_string = sys.argv[1] -param_string = sys.argv[2] -codec = parse_codecs_zarrita(codec_string, param_string) - -store = zarrita.LocalStore(sys.argv[3]) -testdata = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) - -a = zarrita.Array.create( - store / 'read_from_zarrita' / codec_string / param_string, - shape=(16, 16, 16), - chunk_shape=(2, 4, 8), - dtype='int32', - codecs=codec, - attributes={'answer': 42} -) -a[:, :] = testdata diff --git a/testdata/v2_sample/.zattrs b/testdata/v2_sample/.zattrs new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/testdata/v2_sample/.zattrs @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/testdata/v2_sample/.zgroup b/testdata/v2_sample/.zgroup new file mode 100644 index 0000000..cab13da --- /dev/null +++ b/testdata/v2_sample/.zgroup @@ -0,0 +1,3 @@ +{ + "zarr_format": 2 +} \ No newline at end of file diff --git a/testdata/v2_sample/bool/.zarray b/testdata/v2_sample/bool/.zarray new file mode 100644 index 0000000..98c3d4c --- /dev/null +++ b/testdata/v2_sample/bool/.zarray @@ -0,0 +1,25 @@ +{ + "shape": [ + 16, + 16, + 16 + ], + "chunks": [ + 2, + 4, + 8 + ], + "dtype": "|b1", + "fill_value": false, + "order": "C", + "filters": null, + "dimension_separator": ".", + "compressor": { + "id": "blosc", + "cname": "blosclz", + "clevel": 3, + "shuffle": 1, + "blocksize": 0 + }, + "zarr_format": 2 +} \ No newline at end of file diff --git a/testdata/v2_sample/bool/.zattrs b/testdata/v2_sample/bool/.zattrs new file mode 100644 index 0000000..d98f0ae --- /dev/null +++ b/testdata/v2_sample/bool/.zattrs @@ -0,0 +1,3 @@ +{ + "answer": 42 +} \ No newline at end of file diff --git a/testdata/v2_sample/bool/0.0.0 b/testdata/v2_sample/bool/0.0.0 new file mode 100644 index 0000000..3d2e66b Binary files /dev/null and b/testdata/v2_sample/bool/0.0.0 differ diff --git a/testdata/v2_sample/double/.zarray b/testdata/v2_sample/double/.zarray new file mode 100644 index 0000000..a614af6 --- /dev/null +++ b/testdata/v2_sample/double/.zarray @@ -0,0 +1,25 @@ +{ + "shape": [ + 16, + 16, + 16 + ], + "chunks": [ + 2, + 4, + 8 + ], + "dtype": "