From 69ff8f032c19b9c1b3d9fdc4bd4a3f473a58dbd0 Mon Sep 17 00:00:00 2001 From: James Mudd Date: Mon, 31 Aug 2020 22:06:45 +0100 Subject: [PATCH 1/8] Initially working patch --- .../jhdf/dataset/ContiguousDatasetImpl.java | 3 +- .../java/io/jhdf/dataset/DatasetLoader.java | 4 +- .../dataset/chunked/ChunkedDatasetV3.java | 6 +- .../object/message/DataLayoutMessage.java | 117 ++++++++++++++---- 4 files changed, 99 insertions(+), 31 deletions(-) diff --git a/jhdf/src/main/java/io/jhdf/dataset/ContiguousDatasetImpl.java b/jhdf/src/main/java/io/jhdf/dataset/ContiguousDatasetImpl.java index c2590765..825a5f0c 100644 --- a/jhdf/src/main/java/io/jhdf/dataset/ContiguousDatasetImpl.java +++ b/jhdf/src/main/java/io/jhdf/dataset/ContiguousDatasetImpl.java @@ -36,8 +36,7 @@ public ByteBuffer getDataBuffer() { } try { - ByteBuffer data = hdfFc.map(contiguousDataLayoutMessage.getAddress(), - contiguousDataLayoutMessage.getSize()); + ByteBuffer data = hdfFc.map(contiguousDataLayoutMessage.getAddress(), getSizeInBytes()); convertToCorrectEndiness(data); return data; } catch (Exception e) { diff --git a/jhdf/src/main/java/io/jhdf/dataset/DatasetLoader.java b/jhdf/src/main/java/io/jhdf/dataset/DatasetLoader.java index 7609dff2..7bf0d9ec 100644 --- a/jhdf/src/main/java/io/jhdf/dataset/DatasetLoader.java +++ b/jhdf/src/main/java/io/jhdf/dataset/DatasetLoader.java @@ -17,7 +17,7 @@ import io.jhdf.dataset.chunked.ChunkedDatasetV4; import io.jhdf.exceptions.HdfException; import io.jhdf.object.message.DataLayoutMessage; -import io.jhdf.object.message.DataLayoutMessage.ChunkedDataLayoutMessageV3; +import io.jhdf.object.message.DataLayoutMessage.ChunkedDataLayoutMessage; import io.jhdf.object.message.DataLayoutMessage.ChunkedDataLayoutMessageV4; import io.jhdf.object.message.DataLayoutMessage.CompactDataLayoutMessage; import io.jhdf.object.message.DataLayoutMessage.ContiguousDataLayoutMessage; @@ -42,7 +42,7 @@ public static Dataset createDataset(HdfFileChannel hdfFc, ObjectHeader oh, Strin } else if (dlm instanceof ContiguousDataLayoutMessage) { return new ContiguousDatasetImpl(hdfFc, address, name, parent, oh); - } else if (dlm instanceof ChunkedDataLayoutMessageV3) { + } else if (dlm instanceof ChunkedDataLayoutMessage) { return new ChunkedDatasetV3(hdfFc, address, name, parent, oh); } else if (dlm instanceof ChunkedDataLayoutMessageV4) { diff --git a/jhdf/src/main/java/io/jhdf/dataset/chunked/ChunkedDatasetV3.java b/jhdf/src/main/java/io/jhdf/dataset/chunked/ChunkedDatasetV3.java index 0ee1d4a2..11ad26be 100644 --- a/jhdf/src/main/java/io/jhdf/dataset/chunked/ChunkedDatasetV3.java +++ b/jhdf/src/main/java/io/jhdf/dataset/chunked/ChunkedDatasetV3.java @@ -16,7 +16,7 @@ import io.jhdf.btree.BTreeV1; import io.jhdf.btree.BTreeV1Data; import io.jhdf.exceptions.HdfException; -import io.jhdf.object.message.DataLayoutMessage.ChunkedDataLayoutMessageV3; +import io.jhdf.object.message.DataLayoutMessage.ChunkedDataLayoutMessage; import org.apache.commons.lang3.concurrent.ConcurrentException; import org.apache.commons.lang3.concurrent.LazyInitializer; import org.slf4j.Logger; @@ -39,14 +39,14 @@ public class ChunkedDatasetV3 extends ChunkedDatasetBase { private static final Logger logger = LoggerFactory.getLogger(ChunkedDatasetV3.class); - private final ChunkedDataLayoutMessageV3 layoutMessage; + private final ChunkedDataLayoutMessage layoutMessage; private final ChunkLookupLazyInitializer chunkLookupLazyInitializer; public ChunkedDatasetV3(HdfFileChannel hdfFc, long address, String name, Group parent, ObjectHeader oh) { super(hdfFc, address, name, parent, oh); - layoutMessage = oh.getMessageOfType(ChunkedDataLayoutMessageV3.class); + layoutMessage = oh.getMessageOfType(ChunkedDataLayoutMessage.class); chunkLookupLazyInitializer = new ChunkLookupLazyInitializer(); } diff --git a/jhdf/src/main/java/io/jhdf/object/message/DataLayoutMessage.java b/jhdf/src/main/java/io/jhdf/object/message/DataLayoutMessage.java index 9f7ac1ad..3ef9c899 100644 --- a/jhdf/src/main/java/io/jhdf/object/message/DataLayoutMessage.java +++ b/jhdf/src/main/java/io/jhdf/object/message/DataLayoutMessage.java @@ -9,6 +9,7 @@ */ package io.jhdf.object.message; +import io.jhdf.Constants; import io.jhdf.Superblock; import io.jhdf.Utils; import io.jhdf.exceptions.HdfException; @@ -29,28 +30,78 @@ public DataLayoutMessage(BitSet flags) { public static DataLayoutMessage createDataLayoutMessage(ByteBuffer bb, Superblock sb, BitSet flags) { final byte version = bb.get(); - if (version != 3 && version != 4) { - throw new UnsupportedHdfException( - "Only v3 and v4 data layout messages are supported. Detected version = " + version); - } +// if (version != 3 && version != 4) { +// throw new UnsupportedHdfException( +// "Only v3 and v4 data layout messages are supported. Detected version = " + version); +// } + + if(version == 1 || version == 2) { + byte dimensionality = bb.get(); // for chunked is +1 than actual dims + final byte layoutClass = bb.get(); + + bb.position(bb.position() + 5); // skip reserved bytes + + final long dataAddress; + if (layoutClass != 0) { // not compact + dataAddress = Utils.readBytesAsUnsignedLong(bb, sb.getSizeOfOffsets()); + } else { + dataAddress = Constants.UNDEFINED_ADDRESS; + } + + // Now skip the dims +// final int dimFieldBytes = 4 * dimensionality; +// bb.position(bb.position() + dimFieldBytes); + if (layoutClass == 2) { + dimensionality--; + } + + int[] dimensions = new int[dimensionality]; + for (int i = 0; i < dimensions.length; i++) { + dimensions[i] = Utils.readBytesAsUnsignedInt(bb, 4); + } - final byte layoutClass = bb.get(); + if (layoutClass == 1) { // Contiguous + return new ContiguousDataLayoutMessage(flags, dataAddress, -1); - switch (layoutClass) { - case 0: // Compact Storage - return new CompactDataLayoutMessage(bb, flags); - case 1: // Contiguous Storage - return new ContiguousDataLayoutMessage(bb, sb, flags); - case 2: // Chunked Storage - if (version == 3) { - return new ChunkedDataLayoutMessageV3(bb, sb, flags); - } else { // v4 - return new ChunkedDataLayoutMessageV4(bb, sb, flags); + } else if (layoutClass == 2) { // Chunked + final int dataElementSize = Utils.readBytesAsUnsignedInt(bb, 4); + return new ChunkedDataLayoutMessage(flags, dataAddress, dataElementSize, dimensions); + } + + return null; +// switch (layoutClass) { +// case 0: // Compact Storage +// return new CompactDataLayoutMessage(bb, flags); +// case 1: // Contiguous Storage +// return new ContiguousDataLayoutMessage(bb, sb, flags); +// case 2: // Chunked Storage +// if (version == 3) { +// return new ChunkedDataLayoutMessageV3(bb, sb, flags); +// } else { // v4 +// return new ChunkedDataLayoutMessageV4(bb, sb, flags); +// } +// } + + + } else { + final byte layoutClass = bb.get(); + + switch (layoutClass) { + case 0: // Compact Storage + return new CompactDataLayoutMessage(bb, flags); + case 1: // Contiguous Storage + return new ContiguousDataLayoutMessage(bb, sb, flags); + case 2: // Chunked Storage + if (version == 3) { + return new ChunkedDataLayoutMessage(bb, sb, flags); + } else { // v4 + return new ChunkedDataLayoutMessageV4(bb, sb, flags); + } + case 3: // Virtual storage + throw new UnsupportedHdfException("Virtual storage is not supported"); + default: + throw new UnsupportedHdfException("Unknown storage layout " + layoutClass); } - case 3: // Virtual storage - throw new UnsupportedHdfException("Virtual storage is not supported"); - default: - throw new UnsupportedHdfException("Unknown storage layout " + layoutClass); } } @@ -58,6 +109,11 @@ public static class CompactDataLayoutMessage extends DataLayoutMessage { private final ByteBuffer dataBuffer; + public CompactDataLayoutMessage(BitSet flags, ByteBuffer dataBuffer) { + super(flags); + this.dataBuffer = dataBuffer; + } + private CompactDataLayoutMessage(ByteBuffer bb, BitSet flags) { super(flags); final int compactDataSize = Utils.readBytesAsUnsignedInt(bb, 2); @@ -79,6 +135,12 @@ public static class ContiguousDataLayoutMessage extends DataLayoutMessage { private final long address; private final long size; + public ContiguousDataLayoutMessage(BitSet flags, long address, long size) { + super(flags); + this.address = address; + this.size = size; + } + private ContiguousDataLayoutMessage(ByteBuffer bb, Superblock sb, BitSet flags) { super(flags); address = Utils.readBytesAsUnsignedLong(bb, sb.getSizeOfOffsets()); @@ -99,16 +161,23 @@ public long getSize() { } } - public static class ChunkedDataLayoutMessageV3 extends DataLayoutMessage { + public static class ChunkedDataLayoutMessage extends DataLayoutMessage { - private final long address; + private final long bTreeAddress; private final int size; private final int[] chunkDimensions; - private ChunkedDataLayoutMessageV3(ByteBuffer bb, Superblock sb, BitSet flags) { + public ChunkedDataLayoutMessage(BitSet flags, long bTreeAddress, int size, int[] chunkDimensions) { + super(flags); + this.bTreeAddress = bTreeAddress; + this.size = size; + this.chunkDimensions = chunkDimensions; + } + + private ChunkedDataLayoutMessage(ByteBuffer bb, Superblock sb, BitSet flags) { super(flags); final int chunkDimensionality = bb.get() - 1; - address = Utils.readBytesAsUnsignedLong(bb, sb.getSizeOfOffsets()); + bTreeAddress = Utils.readBytesAsUnsignedLong(bb, sb.getSizeOfOffsets()); chunkDimensions = new int[chunkDimensionality]; for (int i = 0; i < chunkDimensions.length; i++) { chunkDimensions[i] = Utils.readBytesAsUnsignedInt(bb, 4); @@ -122,7 +191,7 @@ public DataLayout getDataLayout() { } public long getBTreeAddress() { - return address; + return bTreeAddress; } public int getSize() { From 05809146a3acbaa1cc106ad18810b1701edd4136 Mon Sep 17 00:00:00 2001 From: James Mudd Date: Tue, 1 Sep 2020 16:06:46 +0100 Subject: [PATCH 2/8] Cleanup of impl --- .../object/message/DataLayoutMessage.java | 194 +++++++++--------- 1 file changed, 97 insertions(+), 97 deletions(-) diff --git a/jhdf/src/main/java/io/jhdf/object/message/DataLayoutMessage.java b/jhdf/src/main/java/io/jhdf/object/message/DataLayoutMessage.java index 3ef9c899..7e0116fa 100644 --- a/jhdf/src/main/java/io/jhdf/object/message/DataLayoutMessage.java +++ b/jhdf/src/main/java/io/jhdf/object/message/DataLayoutMessage.java @@ -30,78 +30,75 @@ public DataLayoutMessage(BitSet flags) { public static DataLayoutMessage createDataLayoutMessage(ByteBuffer bb, Superblock sb, BitSet flags) { final byte version = bb.get(); -// if (version != 3 && version != 4) { -// throw new UnsupportedHdfException( -// "Only v3 and v4 data layout messages are supported. Detected version = " + version); -// } - - if(version == 1 || version == 2) { - byte dimensionality = bb.get(); // for chunked is +1 than actual dims - final byte layoutClass = bb.get(); - - bb.position(bb.position() + 5); // skip reserved bytes - - final long dataAddress; - if (layoutClass != 0) { // not compact - dataAddress = Utils.readBytesAsUnsignedLong(bb, sb.getSizeOfOffsets()); - } else { - dataAddress = Constants.UNDEFINED_ADDRESS; - } + switch (version) { + case 1: + case 2: + return readV1V2Message(bb, sb, flags); + case 3: + case 4: + return readV3V4Message(bb, sb, flags, version); + default: + throw new UnsupportedHdfException("Unsupported data layout message version detected. Detected version = " + version); + } + } - // Now skip the dims -// final int dimFieldBytes = 4 * dimensionality; -// bb.position(bb.position() + dimFieldBytes); - if (layoutClass == 2) { - dimensionality--; - } + private static DataLayoutMessage readV1V2Message(ByteBuffer bb, Superblock sb, BitSet flags) { + byte dimensionality = bb.get(); // for chunked is +1 than actual dims - int[] dimensions = new int[dimensionality]; - for (int i = 0; i < dimensions.length; i++) { - dimensions[i] = Utils.readBytesAsUnsignedInt(bb, 4); - } + final byte layoutClass = bb.get(); - if (layoutClass == 1) { // Contiguous - return new ContiguousDataLayoutMessage(flags, dataAddress, -1); + bb.position(bb.position() + 5); // skip reserved bytes - } else if (layoutClass == 2) { // Chunked - final int dataElementSize = Utils.readBytesAsUnsignedInt(bb, 4); - return new ChunkedDataLayoutMessage(flags, dataAddress, dataElementSize, dimensions); - } + final long dataAddress; + if (layoutClass != 0) { // not compact + dataAddress = Utils.readBytesAsUnsignedLong(bb, sb.getSizeOfOffsets()); + } else { + dataAddress = Constants.UNDEFINED_ADDRESS; + } - return null; -// switch (layoutClass) { -// case 0: // Compact Storage -// return new CompactDataLayoutMessage(bb, flags); -// case 1: // Contiguous Storage -// return new ContiguousDataLayoutMessage(bb, sb, flags); -// case 2: // Chunked Storage -// if (version == 3) { -// return new ChunkedDataLayoutMessageV3(bb, sb, flags); -// } else { // v4 -// return new ChunkedDataLayoutMessageV4(bb, sb, flags); -// } -// } + // If chunked value stored is +1 so correct it here + if (layoutClass == 2) { + dimensionality--; + } + int[] dimensions = new int[dimensionality]; + for (int i = 0; i < dimensions.length; i++) { + dimensions[i] = Utils.readBytesAsUnsignedInt(bb, 4); + } - } else { - final byte layoutClass = bb.get(); - - switch (layoutClass) { - case 0: // Compact Storage - return new CompactDataLayoutMessage(bb, flags); - case 1: // Contiguous Storage - return new ContiguousDataLayoutMessage(bb, sb, flags); - case 2: // Chunked Storage - if (version == 3) { - return new ChunkedDataLayoutMessage(bb, sb, flags); - } else { // v4 - return new ChunkedDataLayoutMessageV4(bb, sb, flags); - } - case 3: // Virtual storage - throw new UnsupportedHdfException("Virtual storage is not supported"); - default: - throw new UnsupportedHdfException("Unknown storage layout " + layoutClass); - } + switch (layoutClass) { + case 0: // Compact Storage + final int compactDataSize = Utils.readBytesAsUnsignedInt(bb, 4); + final ByteBuffer compactDataBuffer = Utils.createSubBuffer(bb, compactDataSize); + return new CompactDataLayoutMessage(flags, compactDataBuffer); + case 1: // Contiguous + return new ContiguousDataLayoutMessage(flags, dataAddress, -1L); + case 2: // Chunked + final int dataElementSize = Utils.readBytesAsUnsignedInt(bb, 4); + return new ChunkedDataLayoutMessage(flags, dataAddress, dataElementSize, dimensions); + default: + throw new UnsupportedHdfException("Unknown storage layout " + layoutClass); + } + } + + private static DataLayoutMessage readV3V4Message(ByteBuffer bb, Superblock sb, BitSet flags, byte version) { + final byte layoutClass = bb.get(); + + switch (layoutClass) { + case 0: // Compact Storage + return new CompactDataLayoutMessage(bb, flags); + case 1: // Contiguous Storage + return new ContiguousDataLayoutMessage(bb, sb, flags); + case 2: // Chunked Storage + if (version == 3) { + return new ChunkedDataLayoutMessage(bb, sb, flags); + } else { // v4 + return new ChunkedDataLayoutMessageV4(bb, sb, flags); + } + case 3: // Virtual storage + throw new UnsupportedHdfException("Virtual storage is not supported"); + default: + throw new UnsupportedHdfException("Unknown storage layout " + layoutClass); } } @@ -109,7 +106,7 @@ public static class CompactDataLayoutMessage extends DataLayoutMessage { private final ByteBuffer dataBuffer; - public CompactDataLayoutMessage(BitSet flags, ByteBuffer dataBuffer) { + private CompactDataLayoutMessage(BitSet flags, ByteBuffer dataBuffer) { super(flags); this.dataBuffer = dataBuffer; } @@ -135,7 +132,7 @@ public static class ContiguousDataLayoutMessage extends DataLayoutMessage { private final long address; private final long size; - public ContiguousDataLayoutMessage(BitSet flags, long address, long size) { + private ContiguousDataLayoutMessage(BitSet flags, long address, long size) { super(flags); this.address = address; this.size = size; @@ -156,6 +153,9 @@ public long getAddress() { return address; } + /** + * @return size in bytes if known or -1 otherwise + */ public long getSize() { return size; } @@ -171,7 +171,7 @@ public ChunkedDataLayoutMessage(BitSet flags, long bTreeAddress, int size, int[] super(flags); this.bTreeAddress = bTreeAddress; this.size = size; - this.chunkDimensions = chunkDimensions; + this.chunkDimensions = ArrayUtils.clone(chunkDimensions); } private ChunkedDataLayoutMessage(ByteBuffer bb, Superblock sb, BitSet flags) { @@ -229,7 +229,7 @@ public static class ChunkedDataLayoutMessageV4 extends DataLayoutMessage { private ChunkedDataLayoutMessageV4(ByteBuffer bb, Superblock sb, BitSet flags) { super(flags); - final BitSet chunkedFlags = BitSet.valueOf(new byte[] { bb.get() }); + final BitSet chunkedFlags = BitSet.valueOf(new byte[]{bb.get()}); final int chunkDimensionality = bb.get(); final int dimSizeBytes = bb.get(); @@ -241,37 +241,37 @@ private ChunkedDataLayoutMessageV4(ByteBuffer bb, Superblock sb, BitSet flags) { indexingType = bb.get(); switch (indexingType) { - case 1: // Single Chunk - if (chunkedFlags.get(SINGLE_INDEX_WITH_FILTER)) { - isFilteredSingleChunk = true; - sizeOfFilteredSingleChunk = Utils.readBytesAsUnsignedInt(bb, sb.getSizeOfLengths()); - filterMaskFilteredSingleChunk = BitSet.valueOf(new byte[] { bb.get(), bb.get(), bb.get(), bb.get() }); - } - break; + case 1: // Single Chunk + if (chunkedFlags.get(SINGLE_INDEX_WITH_FILTER)) { + isFilteredSingleChunk = true; + sizeOfFilteredSingleChunk = Utils.readBytesAsUnsignedInt(bb, sb.getSizeOfLengths()); + filterMaskFilteredSingleChunk = BitSet.valueOf(new byte[]{bb.get(), bb.get(), bb.get(), bb.get()}); + } + break; - case 2: // Implicit - break; // There is nothing for this case + case 2: // Implicit + break; // There is nothing for this case - case 3: // Fixed Array - pageBits = bb.get(); - break; + case 3: // Fixed Array + pageBits = bb.get(); + break; - case 4: // Extensible Array - maxBits = bb.get(); - indexElements = bb.get(); - minPointers = bb.get(); - minElements = bb.get(); - pageBits = bb.get(); // This is wrong in the spec says 2 bytes its actually 1 - break; + case 4: // Extensible Array + maxBits = bb.get(); + indexElements = bb.get(); + minPointers = bb.get(); + minElements = bb.get(); + pageBits = bb.get(); // This is wrong in the spec says 2 bytes its actually 1 + break; - case 5: // B tree v2 - nodeSize = bb.getInt(); - splitPercent = bb.get(); - mergePercent = bb.get(); - break; + case 5: // B tree v2 + nodeSize = bb.getInt(); + splitPercent = bb.get(); + mergePercent = bb.get(); + break; - default: - throw new UnsupportedHdfException("Unrecognized chunk indexing type. type=" + indexingType); + default: + throw new UnsupportedHdfException("Unrecognized chunk indexing type. type=" + indexingType); } address = Utils.readBytesAsUnsignedLong(bb, sb.getSizeOfOffsets()); @@ -327,14 +327,14 @@ public int[] getChunkDimensions() { } public int getSizeOfFilteredSingleChunk() { - if(!isFilteredSingleChunk) { + if (!isFilteredSingleChunk) { throw new HdfException("Requested size of filtered single chunk when its not set."); } return sizeOfFilteredSingleChunk; } public BitSet getFilterMaskFilteredSingleChunk() { - if(!isFilteredSingleChunk){ + if (!isFilteredSingleChunk) { throw new HdfException("Requested filter mask of filtered single chunk when its not set."); } return filterMaskFilteredSingleChunk; From 401552ece84c014f046e0621b54f688a29c00bbd Mon Sep 17 00:00:00 2001 From: James Mudd Date: Tue, 1 Sep 2020 21:12:01 +0100 Subject: [PATCH 3/8] Add compact dataset tests --- .../object/message/DataLayoutMessage.java | 4 +- .../hdf5/test_compact_datasets_earliest.hdf5 | Bin 0 -> 12112 bytes .../hdf5/test_compact_datasets_latest.hdf5 | Bin 0 -> 8561 bytes .../resources/scripts/compact_datasets.py | 71 ++++++++++++++++++ 4 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 jhdf/src/test/resources/hdf5/test_compact_datasets_earliest.hdf5 create mode 100644 jhdf/src/test/resources/hdf5/test_compact_datasets_latest.hdf5 create mode 100644 jhdf/src/test/resources/scripts/compact_datasets.py diff --git a/jhdf/src/main/java/io/jhdf/object/message/DataLayoutMessage.java b/jhdf/src/main/java/io/jhdf/object/message/DataLayoutMessage.java index 7e0116fa..a2309528 100644 --- a/jhdf/src/main/java/io/jhdf/object/message/DataLayoutMessage.java +++ b/jhdf/src/main/java/io/jhdf/object/message/DataLayoutMessage.java @@ -114,7 +114,7 @@ private CompactDataLayoutMessage(BitSet flags, ByteBuffer dataBuffer) { private CompactDataLayoutMessage(ByteBuffer bb, BitSet flags) { super(flags); final int compactDataSize = Utils.readBytesAsUnsignedInt(bb, 2); - dataBuffer = Utils.createSubBuffer(bb, compactDataSize); + this.dataBuffer = Utils.createSubBuffer(bb, compactDataSize); } @Override @@ -123,7 +123,7 @@ public DataLayout getDataLayout() { } public ByteBuffer getDataBuffer() { - return dataBuffer; + return dataBuffer.slice(); } } diff --git a/jhdf/src/test/resources/hdf5/test_compact_datasets_earliest.hdf5 b/jhdf/src/test/resources/hdf5/test_compact_datasets_earliest.hdf5 new file mode 100644 index 0000000000000000000000000000000000000000..c213a3f9aec618e8418232d43b5159fb89e2a0ae GIT binary patch literal 12112 zcmeHNPj3@P6o2cD6O)v(0R*8%s?}2t2<+H#j8qBJCg4K_R@;Mb#y~Nw_jFP04?SHLb)0o z=Jk`jN92xv83;62ZnvnXZSHQ7=|1}zv#_@DthV|tZok?Hq}W*=XBJ|B$_qw=AEC%(ib zaOI0aASoZ}bABOUU>;wmXQY@FDT-OqN%DCe$Ahughd{;Y03B6V(H{KF&wUa|o_DP9=q+0BbZ{~8sCTRef=+3TWf(SLw|#Nj{qThEyeuK&9qmwCTG z_WG#z%NNj|cF6JhexMrc!#+0kI2*rvkPIi5@xGmb}`J>!t7F*t%q55`P1g_GERfB z$4Q?HUP1c+M}#h2HM+0R;sU(qaT@@Q>(N!kDy7L&Ze#M4=a@X@JSI>1kI7RmWb%|3 znLM@5Hg0OZvq|*51Z->@nNSdshpAI)K@BDeOcv`B)97q zRIZtv$1Bcd4-#zysJ6;5a0p9Xj9?bi!#!P9|YabXBx!`TV6PYP=dZ!K)5YAqhgY0oAwh zN#J#i@M>f+fXOy|NZ=oqNh~v1=CORm0w)0cCyF1#mlyBcRb$w*35IRJ=Y0aAWe{D( zuUlYvjxg*&h-6tNEJ+gV7Yw2OLI;H2D|D^UL7@*NiXZF!uMS?>3gzH@*1&^|7yn>V z9w$b0g;-Uv-vN2wOg1^V_&u+aC;R{m{+q36-uH5;<<3oy^&pY;?px_xHr+Rru@L&h z3IqftG@tu^`eb((%OIr_g=drFMYi?xZ(@rm`3X-FYdpjjSn{jmVYc|%$0KZEwvR{I zqHP~hXHrW1>PT7 zYo{Jxl+9S`Kpk%eybocbj>iBmD+ Date: Tue, 8 Sep 2020 19:43:57 +0100 Subject: [PATCH 4/8] Add HDF v1.4 test files --- jhdf/src/test/resources/hdf5/hdf_v14_test1.hdf5 | Bin 0 -> 7072 bytes jhdf/src/test/resources/hdf5/hdf_v14_test2.hdf5 | Bin 0 -> 9936 bytes 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 jhdf/src/test/resources/hdf5/hdf_v14_test1.hdf5 create mode 100644 jhdf/src/test/resources/hdf5/hdf_v14_test2.hdf5 diff --git a/jhdf/src/test/resources/hdf5/hdf_v14_test1.hdf5 b/jhdf/src/test/resources/hdf5/hdf_v14_test1.hdf5 new file mode 100644 index 0000000000000000000000000000000000000000..9c6815e619d3288919f447fc6c968a05016b2ad4 GIT binary patch literal 7072 zcmeI%ZLAzs8Nl(gz3uJLQrMP-PH!i=U59Zq*$zAJFT0yv(wCHCvGm0>lG=(jqK4L{ zG@6L0s1ag8v>Hj&grs6Es+MS@BV6pe<&evrggY`_?mn5-{p1jYKCJ^!n60~(;; zWRkPbK07mKe{=SkvwP3IbHm#8OBT&6QhKv2HDA@$)Fd$v=h^OC`Qm)`kO$|PXEk29 zm$;j_Vcpf6_0ul7O&rhgXc%WmGy!!nM; zLhrOV;zdD|bWi%0nlEjEG+nt1rRftclBVBcv9yb&Es=JKv`eLF;8JPJq+KTMLiVLE z=R&!3Uzba}5Z5@{*A>#=YaOru>}$F7_gcn&ZeO}cUhU`jQhlwE{$5Q@yj)*;c73Jw z_22r^cgZWM)&JF(v-P^SuGRPO>|OI)?l*a#m)vIO`A769y}tcx1m@0bQ!;<)iDK!h zA!9pI$zqk9=SzL={PqTV(jS=KX6MD-bLK^}_ur4hne6Fjoi(e6J~2M(ceh734lDA} zjhjyW@#x=&Yj}PAc>1mfA9{Sak^P%K@`DW@{mJ3sW{%sk;NjYly;lvla{TtU+`DV~ z+!u#$>|6JLliQd9 z!|}ph`%b;_&k}!h?arU>88L(slbav=c`0$?l=#(auD|`5#EEyLYP!Rf5+`nn^7f|> zNu2m4K6&~tpMGJa%ymc{9sgLeSK{P)BpyAy`0TGFPOfXDrk3sf(p{S+POeYlhc~{d zJRxy%of3b2g*S;@`kP_O-I*^>uWjJS~` z8eaon178DQ178DQ178DQ178DQ178DQ178DQ178DQ178DQ1E2YMqeeD98=sBO#%JTR z@!9xnd^SECpN-GPXXCT++4yXHHa;7ljn7e&ZRFr{@HzM#d=5SbpM%fA=iqbjIrtoW z4n7B;gU`X|;B)Xf_?q~dYRSs|2k#y=aW>VPw`|*eOmb6?O?9;t%;S=qx@;;buk0Dg zO?@`i!E-OHIxo4Y)26y-_1PsWB{%ijRLA!G@UMmBrf!?+(J$Zs%r?nQ{WjH8r|x*- z1CpCMZmQ=ldjEG1jav9x_*(c{_*(c{_*(c{_*(c{_*(c{_*(c{_*(c{_*(c{_*(c{ z_*{H0J{O;h&&B8BbMd+OTzoD*7oUsI#pmL4@wxb1d@ep0pNr4K=i&44dH6hh9zGAB zhtI?3;q&l$_&j_bJ`bOV&%@{8^YD52+W6Y|+W6Y|+W6Y|+W6Y|+W6Y|+W6Y|+W6Y| z+W6Y|+W6Y|+W6Y|e0)AWAD@rU$LHho@%i|Cd_F!OpO4SS=i~G7`S^T%K0Y6xkFSHT zgRg_HgRg_HgRg_HgRg_HgRg_HgRg_HgRg_HgRg_HgRg_HgD=1r;0y2t_yT+Zz5ri< zFTfYz3-AT_0(=3!0AGMFz!%^P@CEp~__}KHdv4Ul*;SY6Uv83{@4c>S>IqA7GcR_P zp5FFLZsy0Xs{VG{=al4Tp6sd(SB$6cmfXyjU9~-W=dF(9X5Q>7{X_AX<<;4API_zHXlz5-u?ufSK} zEASQg3VbEwSK=%2mH0}0CB710iLb<0;w$l$_)2^wz7k)Fuf$j4EAf^1Dtr~b3SWh< z!dKy|@KyLKd=!BQJ+(rGH32!eN8Xs>m>kltX|AF W3l?+#UoiLlF6%COJx-rbs{RT7LeY@` literal 0 HcmV?d00001 diff --git a/jhdf/src/test/resources/hdf5/hdf_v14_test2.hdf5 b/jhdf/src/test/resources/hdf5/hdf_v14_test2.hdf5 new file mode 100644 index 0000000000000000000000000000000000000000..7681231abe2363c25a3cd2fa2648bf8bec61d697 GIT binary patch literal 9936 zcmeHNJ#W-N5FOto#N-PfP=xU1@FmJS5-B+eDMrens~`dj$}57DG7|X#DMLp|NlC#E zproXvOi4jONl6!GcITb&oGnqfILKJqo1NX~=k@dWG-LbCaOd&T{OUZ(mZmgA8MT8G ze@=D#davvQ@~IVji4MF|J5oCwJld6RSN#>Wq>!(Ube2rVG%ruxci}(v|>cZ8-wVw&La-tFDp;4 z-|u%fyPLiK`g(7RNa8o4bRvHDl&8*ozY=FGZr1EDT&>1oy?}qL!Ce4AZFaTHXO9Q? zVGYjWPik;>eMjbe!DH9=@ETSSEqH*NDNnoS3b+EUfGgk%xB{-gKcm3&Cr@`Iw@r1$ zKeWE8I|0-`C(Ct+vpJs7TxGt0p!xpD$YS&U8#<3X6F%~PE|yCSG2dUD$oIdLP?OGk z&1)?Vc`)|azaD$Q*?Hi)x=@R=_+JKx^8^p<=bPo@0%!S-^D+*3Ie1`q*Q#-rZ{=1Q zhk6S^COXz?PxU@X&o-5!AxB{+#E8q&a0)HwX zDb)hEMQ-v|Lo)fM$Cxst!%|Ji>m58Um51C-jY;Zrj@ukJNq^3BYkI6{U8?vbb(ONJ z!|fusOWc|sW6J;HdNi#|6`yED+dqS$A{r_*!7fk$>-e%OP4{6onUc>_P5xBdnMn*)mg literal 0 HcmV?d00001 From a62c23fa85b0c83f083afd35c9ea2a1f19f47bce Mon Sep 17 00:00:00 2001 From: James Mudd Date: Tue, 8 Sep 2020 21:30:35 +0100 Subject: [PATCH 5/8] Add support for Old Object Modification Time Message --- .../java/io/jhdf/object/message/Message.java | 2 + .../OldObjectModificationTimeMessage.java | 67 +++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 jhdf/src/main/java/io/jhdf/object/message/OldObjectModificationTimeMessage.java diff --git a/jhdf/src/main/java/io/jhdf/object/message/Message.java b/jhdf/src/main/java/io/jhdf/object/message/Message.java index 43861589..e9be496d 100644 --- a/jhdf/src/main/java/io/jhdf/object/message/Message.java +++ b/jhdf/src/main/java/io/jhdf/object/message/Message.java @@ -115,6 +115,8 @@ private static Message readMessage(ByteBuffer bb, Superblock sb, int messageType return new AttributeMessage(bb, sb, flags); case 13: // 0x000D return new ObjectCommentMessage(bb, flags); + case 14: // 0x000E + return new OldObjectModificationTimeMessage(bb, flags); case 16: // 0x0010 return new ObjectHeaderContinuationMessage(bb, sb, flags); case 17: // 0x0011 diff --git a/jhdf/src/main/java/io/jhdf/object/message/OldObjectModificationTimeMessage.java b/jhdf/src/main/java/io/jhdf/object/message/OldObjectModificationTimeMessage.java new file mode 100644 index 00000000..96e62662 --- /dev/null +++ b/jhdf/src/main/java/io/jhdf/object/message/OldObjectModificationTimeMessage.java @@ -0,0 +1,67 @@ +/* + * This file is part of jHDF. A pure Java library for accessing HDF5 files. + * + * http://jhdf.io + * + * Copyright (c) 2020 James Mudd + * + * MIT License see 'LICENSE' file + */ +package io.jhdf.object.message; + +import io.jhdf.Utils; + +import java.nio.ByteBuffer; +import java.time.LocalDateTime; +import java.util.BitSet; + +import static java.lang.Integer.parseInt; +import static java.nio.charset.StandardCharsets.US_ASCII; + +/** + *

+ * Old Object Modification Time Message + *

+ * + *

+ * Format + * Spec + *

+ * + * @author James Mudd + */ +public class OldObjectModificationTimeMessage extends Message { + + final LocalDateTime modificationTime; + + public OldObjectModificationTimeMessage(ByteBuffer bb, BitSet flags) { + super(flags); + + final ByteBuffer yearBuffer = Utils.createSubBuffer(bb, 4); + final int year = parseInt(US_ASCII.decode(yearBuffer).toString()); + + final ByteBuffer monthBuffer = Utils.createSubBuffer(bb, 2); + final int month = parseInt(US_ASCII.decode(monthBuffer).toString()); + + final ByteBuffer dayBuffer = Utils.createSubBuffer(bb, 2); + final int day = parseInt(US_ASCII.decode(dayBuffer).toString()); + + final ByteBuffer hourBuffer = Utils.createSubBuffer(bb, 2); + final int hour = parseInt(US_ASCII.decode(hourBuffer).toString()); + + final ByteBuffer minuteBuffer = Utils.createSubBuffer(bb, 2); + final int minute = parseInt(US_ASCII.decode(minuteBuffer).toString()); + + final ByteBuffer secondBuffer = Utils.createSubBuffer(bb, 2); + final int second = parseInt(US_ASCII.decode(secondBuffer).toString()); + + final ByteBuffer reservedBuffer = Utils.createSubBuffer(bb, 2); + + this.modificationTime = LocalDateTime.of(year, month, day, hour, minute, second); + } + + public LocalDateTime getModifiedTime() { + return modificationTime; + } +} From 860a1010ce7c03b58c39caa15cd2aedc2c3710fa Mon Sep 17 00:00:00 2001 From: James Mudd Date: Wed, 9 Sep 2020 20:17:31 +0100 Subject: [PATCH 6/8] Skip reserved bytes --- .../jhdf/object/message/OldObjectModificationTimeMessage.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jhdf/src/main/java/io/jhdf/object/message/OldObjectModificationTimeMessage.java b/jhdf/src/main/java/io/jhdf/object/message/OldObjectModificationTimeMessage.java index 96e62662..42b7f8a5 100644 --- a/jhdf/src/main/java/io/jhdf/object/message/OldObjectModificationTimeMessage.java +++ b/jhdf/src/main/java/io/jhdf/object/message/OldObjectModificationTimeMessage.java @@ -56,7 +56,8 @@ public OldObjectModificationTimeMessage(ByteBuffer bb, BitSet flags) { final ByteBuffer secondBuffer = Utils.createSubBuffer(bb, 2); final int second = parseInt(US_ASCII.decode(secondBuffer).toString()); - final ByteBuffer reservedBuffer = Utils.createSubBuffer(bb, 2); + // Skip reserved bytes + bb.position(bb.position() + 2); this.modificationTime = LocalDateTime.of(year, month, day, hour, minute, second); } From 88fe228208931eff33512ad05ad18460922119e4 Mon Sep 17 00:00:00 2001 From: James Mudd Date: Wed, 9 Sep 2020 21:25:29 +0100 Subject: [PATCH 7/8] Improve empty dataset detection for contiguous datasets --- .../java/io/jhdf/dataset/ContiguousDatasetImpl.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/jhdf/src/main/java/io/jhdf/dataset/ContiguousDatasetImpl.java b/jhdf/src/main/java/io/jhdf/dataset/ContiguousDatasetImpl.java index 825a5f0c..32d64df3 100644 --- a/jhdf/src/main/java/io/jhdf/dataset/ContiguousDatasetImpl.java +++ b/jhdf/src/main/java/io/jhdf/dataset/ContiguousDatasetImpl.java @@ -22,16 +22,17 @@ public class ContiguousDatasetImpl extends DatasetBase implements ContiguousDataset { + final ContiguousDataLayoutMessage contiguousDataLayoutMessage; + public ContiguousDatasetImpl(HdfFileChannel hdfFc, long address, String name, Group parent, ObjectHeader oh) { super(hdfFc, address, name, parent, oh); + this.contiguousDataLayoutMessage = getHeaderMessage(ContiguousDataLayoutMessage.class); } @Override public ByteBuffer getDataBuffer() { - ContiguousDataLayoutMessage contiguousDataLayoutMessage = getHeaderMessage(ContiguousDataLayoutMessage.class); - // Check for empty dataset - if (contiguousDataLayoutMessage.getAddress() == UNDEFINED_ADDRESS) { + if (isEmpty()) { return null; } @@ -51,8 +52,11 @@ public ByteBuffer getBuffer() { @Override public long getDataAddress() { - ContiguousDataLayoutMessage contiguousDataLayoutMessage = getHeaderMessage(ContiguousDataLayoutMessage.class); return contiguousDataLayoutMessage.getAddress(); } + @Override + public boolean isEmpty() { + return contiguousDataLayoutMessage.getAddress() == UNDEFINED_ADDRESS; + } } From 670d6d6154bd0ce00d13e20c3e17cc1742d8a91c Mon Sep 17 00:00:00 2001 From: James Mudd Date: Wed, 9 Sep 2020 21:42:18 +0100 Subject: [PATCH 8/8] Remove now unneeded empty check --- .../src/main/java/io/jhdf/dataset/ContiguousDatasetImpl.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/jhdf/src/main/java/io/jhdf/dataset/ContiguousDatasetImpl.java b/jhdf/src/main/java/io/jhdf/dataset/ContiguousDatasetImpl.java index 32d64df3..3a6c8da9 100644 --- a/jhdf/src/main/java/io/jhdf/dataset/ContiguousDatasetImpl.java +++ b/jhdf/src/main/java/io/jhdf/dataset/ContiguousDatasetImpl.java @@ -31,11 +31,6 @@ public ContiguousDatasetImpl(HdfFileChannel hdfFc, long address, String name, Gr @Override public ByteBuffer getDataBuffer() { - // Check for empty dataset - if (isEmpty()) { - return null; - } - try { ByteBuffer data = hdfFc.map(contiguousDataLayoutMessage.getAddress(), getSizeInBytes()); convertToCorrectEndiness(data);