diff --git a/jhdf/src/main/java/io/jhdf/Utils.java b/jhdf/src/main/java/io/jhdf/Utils.java index 5ef780e0..ecbee250 100644 --- a/jhdf/src/main/java/io/jhdf/Utils.java +++ b/jhdf/src/main/java/io/jhdf/Utils.java @@ -424,4 +424,22 @@ public static void writeIntToBits(int value, BitSet bits, int start, int length) bits.set(start + i, bi.testBit(i)); } } + + public static Object[] flatten(Object data) { + List flat = new ArrayList<>(); + flattenInternal(data, flat); + return flat.toArray(); + } + + private static void flattenInternal(Object data, List flat) { + int length = Array.getLength(data); + for (int i = 0; i < length; i++) { + Object element = Array.get(data, i); + if (element.getClass().isArray()) { + flattenInternal(element, flat); + } else { + flat.add(element); + } + } + } } diff --git a/jhdf/src/main/java/io/jhdf/WritableDatasetImpl.java b/jhdf/src/main/java/io/jhdf/WritableDatasetImpl.java index 024c44fb..0083bb85 100644 --- a/jhdf/src/main/java/io/jhdf/WritableDatasetImpl.java +++ b/jhdf/src/main/java/io/jhdf/WritableDatasetImpl.java @@ -13,6 +13,7 @@ import io.jhdf.api.Group; import io.jhdf.api.NodeType; import io.jhdf.api.WritiableDataset; +import io.jhdf.exceptions.HdfWritingException; import io.jhdf.exceptions.UnsupportedHdfException; import io.jhdf.filter.PipelineFilterWithData; import io.jhdf.object.datatype.DataType; @@ -36,6 +37,7 @@ import java.util.Collections; import java.util.List; +import static io.jhdf.Utils.flatten; import static io.jhdf.Utils.stripLeadingIndex; public class WritableDatasetImpl extends AbstractWritableNode implements WritiableDataset { @@ -56,17 +58,18 @@ public WritableDatasetImpl(Object data, String name, Group parent) { @Override public long getSize() { - return 0; + return dataSpace.getTotalLength(); } @Override public long getSizeInBytes() { - return 0; + return getSize() * dataType.getSize(); } @Override public long getStorageInBytes() { - return 0; + // As there is no compression this is correct ATM + return getSizeInBytes(); } @Override @@ -76,12 +79,15 @@ public int[] getDimensions() { @Override public boolean isScalar() { - return false; + if (isEmpty()) { + return false; + } + return getDimensions().length == 0; } @Override public boolean isEmpty() { - return false; + return data == null; } @Override @@ -96,27 +102,28 @@ public boolean isVariableLength() { @Override public long[] getMaxSize() { - return new long[0]; + return dataSpace.getMaxSizes(); } @Override public DataLayout getDataLayout() { - return null; + // ATM we only support contiguous + return DataLayout.CONTIGUOUS; } @Override public Object getData() { - return null; + return data; } @Override public Object getDataFlat() { - return null; + return flatten(data); } @Override public Object getData(long[] sliceOffset, int[] sliceDimensions) { - return null; + throw new HdfWritingException("Slicing a writable dataset not supported"); } @Override @@ -136,6 +143,7 @@ public Object getFillValue() { @Override public List getFilters() { + // ATM no filters support return Collections.emptyList(); } @@ -151,22 +159,22 @@ public boolean isGroup() { @Override public File getFile() { - return null; + return getParent().getFile(); } @Override public Path getFileAsPath() { - return null; + return getParent().getFileAsPath(); } @Override public HdfFile getHdfFile() { - return null; + return getParent().getHdfFile(); } @Override public long getAddress() { - return 0; + throw new HdfWritingException("Address not known until written"); } @Override diff --git a/jhdf/src/main/java/io/jhdf/dataset/DatasetBase.java b/jhdf/src/main/java/io/jhdf/dataset/DatasetBase.java index fa27c89b..15efeaf4 100644 --- a/jhdf/src/main/java/io/jhdf/dataset/DatasetBase.java +++ b/jhdf/src/main/java/io/jhdf/dataset/DatasetBase.java @@ -34,7 +34,6 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.util.Arrays; import static java.nio.ByteOrder.LITTLE_ENDIAN; import static java.util.concurrent.TimeUnit.MILLISECONDS; @@ -92,11 +91,7 @@ public int[] getDimensions() { @Override public long[] getMaxSize() { - if (dataSpace.isMaxSizesPresent()) { - return dataSpace.getMaxSizes(); - } else { - return Arrays.stream(getDimensions()).asLongStream().toArray(); - } + return dataSpace.getMaxSizes(); } @Override diff --git a/jhdf/src/main/java/io/jhdf/object/message/DataSpace.java b/jhdf/src/main/java/io/jhdf/object/message/DataSpace.java index 58c1fa97..d9f56fce 100644 --- a/jhdf/src/main/java/io/jhdf/object/message/DataSpace.java +++ b/jhdf/src/main/java/io/jhdf/object/message/DataSpace.java @@ -16,6 +16,7 @@ import org.apache.commons.lang3.ArrayUtils; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.BitSet; import java.util.stream.IntStream; @@ -64,7 +65,7 @@ private DataSpace(ByteBuffer bb, Superblock sb) { maxSizes[i] = Utils.readBytesAsUnsignedLong(bb, sb.getSizeOfLengths()); } } else { - maxSizes = ArrayUtils.EMPTY_LONG_ARRAY; + maxSizes = Arrays.stream(dimensions).asLongStream().toArray(); } // Permutation indices - Note never implemented in HDF library! @@ -83,10 +84,11 @@ public static DataSpace readDataSpace(ByteBuffer bb, Superblock sb) { } public static DataSpace fromObject(Object data) { + int[] dimensions1 = Utils.getDimensions(data); return new DataSpace((byte) 2, false, - Utils.getDimensions(data), - ArrayUtils.EMPTY_LONG_ARRAY, + dimensions1, + Arrays.stream(dimensions1).asLongStream().toArray(), (byte) 1 ); } diff --git a/jhdf/src/test/java/io/jhdf/TestUtils.java b/jhdf/src/test/java/io/jhdf/TestUtils.java index 92b15441..7b39162d 100644 --- a/jhdf/src/test/java/io/jhdf/TestUtils.java +++ b/jhdf/src/test/java/io/jhdf/TestUtils.java @@ -11,11 +11,8 @@ import org.apache.commons.lang3.ArrayUtils; -import java.lang.reflect.Array; import java.net.URL; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; public final class TestUtils { @@ -31,26 +28,8 @@ public static HdfFile loadTestHdfFile(String fileName) throws Exception { return new HdfFile(Paths.get(url.toURI())); } - public static Object[] flatten(Object data) { - List flat = new ArrayList<>(); - flattenInternal(data, flat); - return flat.toArray(); - } - - private static void flattenInternal(Object data, List flat) { - int length = Array.getLength(data); - for (int i = 0; i < length; i++) { - Object element = Array.get(data, i); - if (element.getClass().isArray()) { - flattenInternal(element, flat); - } else { - flat.add(element); - } - } - } - public static String[] toStringArray(Object data) { - return ArrayUtils.toStringArray(flatten(data)); + return ArrayUtils.toStringArray(Utils.flatten(data)); } } diff --git a/jhdf/src/test/java/io/jhdf/WritableDatasetImplTest.java b/jhdf/src/test/java/io/jhdf/WritableDatasetImplTest.java new file mode 100644 index 00000000..703234b0 --- /dev/null +++ b/jhdf/src/test/java/io/jhdf/WritableDatasetImplTest.java @@ -0,0 +1,62 @@ +/* + * This file is part of jHDF. A pure Java library for accessing HDF5 files. + * + * http://jhdf.io + * + * Copyright (c) 2024 James Mudd + * + * MIT License see 'LICENSE' file + */ +package io.jhdf; + +import io.jhdf.api.NodeType; +import io.jhdf.exceptions.HdfException; +import io.jhdf.object.message.DataLayout; +import org.apache.commons.lang3.ArrayUtils; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class WritableDatasetImplTest { + + @Test + void testGettingSize() { + int[] data = new int[] {1,2,3}; + WritableDatasetImpl writableDataset = new WritableDatasetImpl(data, "ints", null); + assertThat(writableDataset.getSize()).isEqualTo(3); + assertThat(writableDataset.getSizeInBytes()).isEqualTo(3 * 4); + assertThat(writableDataset.getStorageInBytes()).isEqualTo(3 * 4); + } + @Test + void testGettingData() { + int[][] data = new int[][] {{1,2,3}, {4,5,6}}; + WritableDatasetImpl writableDataset = new WritableDatasetImpl(data, "ints", null); + assertThat(writableDataset.getData()).isEqualTo(new int[][] {{1,2,3}, {4,5,6}}); + assertThat(writableDataset.getDataFlat()).isEqualTo(ArrayUtils.toObject(new int[] {1,2,3, 4,5,6})); + assertThat(writableDataset.getDimensions()).isEqualTo(new int[]{2,3}); + assertThat(writableDataset.getMaxSize()).isEqualTo(new long[]{2,3}); + assertThat(writableDataset.getJavaType()).isEqualTo(int.class); + assertThat(writableDataset.getDataLayout()).isEqualTo(DataLayout.CONTIGUOUS); + assertThrows(HdfException.class, () -> + writableDataset.getData(new long[] {1, 2}, new int[] {1, 2})); + assertThat(writableDataset.getFillValue()).isNull(); + assertThat(writableDataset.getFilters()).isEmpty(); + assertThat(writableDataset.getDataType()).isNotNull(); + } + + @Test + void testGettingFlags() { + int[][] data = new int[][] {{1,2,3}, {4,5,6}}; + WritableDatasetImpl writableDataset = new WritableDatasetImpl(data, "ints", null); + assertThat(writableDataset.isScalar()).isFalse(); + assertThat(writableDataset.isEmpty()).isFalse(); + assertThat(writableDataset.isLink()).isFalse(); + assertThat(writableDataset.isGroup()).isFalse(); + assertThat(writableDataset.isVariableLength()).isFalse(); + assertThat(writableDataset.isCompound()).isFalse(); + assertThat(writableDataset.isAttributeCreationOrderTracked()).isFalse(); + assertThat(writableDataset.getType()).isEqualTo(NodeType.DATASET); + } + +} diff --git a/jhdf/src/test/java/io/jhdf/dataset/BitfieldDatasetTest.java b/jhdf/src/test/java/io/jhdf/dataset/BitfieldDatasetTest.java index 76f5e6f2..ae2ec343 100644 --- a/jhdf/src/test/java/io/jhdf/dataset/BitfieldDatasetTest.java +++ b/jhdf/src/test/java/io/jhdf/dataset/BitfieldDatasetTest.java @@ -20,7 +20,7 @@ import java.util.stream.Stream; -import static io.jhdf.TestUtils.flatten; +import static io.jhdf.Utils.flatten; import static io.jhdf.TestUtils.loadTestHdfFile; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; diff --git a/jhdf/src/test/java/io/jhdf/dataset/ByteShuffleChunkedDatasetTest.java b/jhdf/src/test/java/io/jhdf/dataset/ByteShuffleChunkedDatasetTest.java index 58a11fa5..ebc2e999 100644 --- a/jhdf/src/test/java/io/jhdf/dataset/ByteShuffleChunkedDatasetTest.java +++ b/jhdf/src/test/java/io/jhdf/dataset/ByteShuffleChunkedDatasetTest.java @@ -24,7 +24,7 @@ import java.util.Collection; import java.util.List; -import static io.jhdf.TestUtils.flatten; +import static io.jhdf.Utils.flatten; import static io.jhdf.TestUtils.loadTestHdfFile; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.arrayContaining; diff --git a/jhdf/src/test/java/io/jhdf/dataset/ChunkedDatasetTest.java b/jhdf/src/test/java/io/jhdf/dataset/ChunkedDatasetTest.java index 41ed829f..a6dc213d 100644 --- a/jhdf/src/test/java/io/jhdf/dataset/ChunkedDatasetTest.java +++ b/jhdf/src/test/java/io/jhdf/dataset/ChunkedDatasetTest.java @@ -20,7 +20,7 @@ import java.util.Arrays; import java.util.Collection; -import static io.jhdf.TestUtils.flatten; +import static io.jhdf.Utils.flatten; import static io.jhdf.Utils.getDimensions; import static io.jhdf.TestUtils.loadTestHdfFile; import static org.hamcrest.MatcherAssert.assertThat; diff --git a/jhdf/src/test/java/io/jhdf/dataset/ChunkedV4DatasetTest.java b/jhdf/src/test/java/io/jhdf/dataset/ChunkedV4DatasetTest.java index fe8a05d2..419896a9 100644 --- a/jhdf/src/test/java/io/jhdf/dataset/ChunkedV4DatasetTest.java +++ b/jhdf/src/test/java/io/jhdf/dataset/ChunkedV4DatasetTest.java @@ -29,7 +29,7 @@ import java.util.List; import java.util.stream.Stream; -import static io.jhdf.TestUtils.flatten; +import static io.jhdf.Utils.flatten; import static io.jhdf.TestUtils.loadTestHdfFile; import static org.apache.commons.lang3.ArrayUtils.toObject; import static org.hamcrest.CoreMatchers.equalTo; diff --git a/jhdf/src/test/java/io/jhdf/dataset/CompactDatasetTest.java b/jhdf/src/test/java/io/jhdf/dataset/CompactDatasetTest.java index ac830266..30208e26 100644 --- a/jhdf/src/test/java/io/jhdf/dataset/CompactDatasetTest.java +++ b/jhdf/src/test/java/io/jhdf/dataset/CompactDatasetTest.java @@ -24,7 +24,7 @@ import java.util.Collection; import java.util.List; -import static io.jhdf.TestUtils.flatten; +import static io.jhdf.Utils.flatten; import static io.jhdf.TestUtils.loadTestHdfFile; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.empty; diff --git a/jhdf/src/test/java/io/jhdf/dataset/CompressedChunkedDatasetTest.java b/jhdf/src/test/java/io/jhdf/dataset/CompressedChunkedDatasetTest.java index 8a25f0f3..131286d0 100644 --- a/jhdf/src/test/java/io/jhdf/dataset/CompressedChunkedDatasetTest.java +++ b/jhdf/src/test/java/io/jhdf/dataset/CompressedChunkedDatasetTest.java @@ -10,7 +10,6 @@ package io.jhdf.dataset; import io.jhdf.HdfFile; -import io.jhdf.TestUtils; import io.jhdf.Utils; import io.jhdf.api.Dataset; import org.junit.jupiter.api.AfterAll; @@ -77,7 +76,7 @@ private Executable createTest(HdfFile hdfFile, String datasetPath, double expect Dataset dataset = hdfFile.getDatasetByPath(datasetPath); Object data = dataset.getData(); assertThat(Utils.getDimensions(data), is(equalTo(new int[]{7, 5}))); - Object[] flatData = TestUtils.flatten(data); + Object[] flatData = Utils.flatten(data); for (int i = 0; i < flatData.length; i++) { // Do element comparison as there are all different primitive numeric types // convert to double diff --git a/jhdf/src/test/java/io/jhdf/dataset/DatasetByAddressTest.java b/jhdf/src/test/java/io/jhdf/dataset/DatasetByAddressTest.java index 9b3211d3..03080851 100644 --- a/jhdf/src/test/java/io/jhdf/dataset/DatasetByAddressTest.java +++ b/jhdf/src/test/java/io/jhdf/dataset/DatasetByAddressTest.java @@ -23,7 +23,7 @@ import java.util.Collection; import java.util.Collections; -import static io.jhdf.TestUtils.flatten; +import static io.jhdf.Utils.flatten; import static io.jhdf.Utils.getDimensions; import static io.jhdf.TestUtils.loadTestHdfFile; import static org.hamcrest.MatcherAssert.assertThat; diff --git a/jhdf/src/test/java/io/jhdf/dataset/EnumDatasetTest.java b/jhdf/src/test/java/io/jhdf/dataset/EnumDatasetTest.java index c4b7e026..187aec18 100644 --- a/jhdf/src/test/java/io/jhdf/dataset/EnumDatasetTest.java +++ b/jhdf/src/test/java/io/jhdf/dataset/EnumDatasetTest.java @@ -24,7 +24,7 @@ import java.util.List; import java.util.stream.Stream; -import static io.jhdf.TestUtils.flatten; +import static io.jhdf.Utils.flatten; import static io.jhdf.TestUtils.loadTestHdfFile; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; diff --git a/jhdf/src/test/java/io/jhdf/dataset/OddDatasetTest.java b/jhdf/src/test/java/io/jhdf/dataset/OddDatasetTest.java index b03e7732..3c1729cc 100644 --- a/jhdf/src/test/java/io/jhdf/dataset/OddDatasetTest.java +++ b/jhdf/src/test/java/io/jhdf/dataset/OddDatasetTest.java @@ -20,7 +20,7 @@ import java.util.Arrays; import java.util.Collection; -import static io.jhdf.TestUtils.flatten; +import static io.jhdf.Utils.flatten; import static io.jhdf.TestUtils.loadTestHdfFile; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; diff --git a/jhdf/src/test/java/io/jhdf/dataset/StringDatasetTest.java b/jhdf/src/test/java/io/jhdf/dataset/StringDatasetTest.java index 5336a72b..2431c099 100644 --- a/jhdf/src/test/java/io/jhdf/dataset/StringDatasetTest.java +++ b/jhdf/src/test/java/io/jhdf/dataset/StringDatasetTest.java @@ -24,7 +24,7 @@ import java.util.Arrays; import java.util.Collection; -import static io.jhdf.TestUtils.flatten; +import static io.jhdf.Utils.flatten; import static io.jhdf.Utils.getDimensions; import static io.jhdf.TestUtils.loadTestHdfFile; import static java.nio.charset.StandardCharsets.UTF_8; diff --git a/jhdf/src/test/java/io/jhdf/dataset/VariableLengthDatasetTest.java b/jhdf/src/test/java/io/jhdf/dataset/VariableLengthDatasetTest.java index c3b3d836..029a4e18 100644 --- a/jhdf/src/test/java/io/jhdf/dataset/VariableLengthDatasetTest.java +++ b/jhdf/src/test/java/io/jhdf/dataset/VariableLengthDatasetTest.java @@ -10,7 +10,7 @@ package io.jhdf.dataset; import io.jhdf.HdfFile; -import io.jhdf.TestUtils; +import io.jhdf.Utils; import io.jhdf.api.Dataset; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -122,7 +122,7 @@ private Executable createTest(HdfFile hdfFile, String datasetPath) { assertThat(Array.getLength(data[2]), is(3)); // Now check the values - Object[] flatData = TestUtils.flatten(data); + Object[] flatData = Utils.flatten(data); for (int i = 0; i < flatData.length; i++) { // Do element comparison as there are all different primitive numeric types // convert to double diff --git a/jhdf/src/test/java/io/jhdf/examples/TestAllFilesBase.java b/jhdf/src/test/java/io/jhdf/examples/TestAllFilesBase.java index e6198925..5e6aab96 100644 --- a/jhdf/src/test/java/io/jhdf/examples/TestAllFilesBase.java +++ b/jhdf/src/test/java/io/jhdf/examples/TestAllFilesBase.java @@ -38,7 +38,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static io.jhdf.TestUtils.flatten; +import static io.jhdf.Utils.flatten; import static io.jhdf.Utils.getDimensions; import static org.apache.commons.lang3.ArrayUtils.toObject; import static org.hamcrest.MatcherAssert.assertThat;