Skip to content

Commit

Permalink
Initial write of chunked dataset + compression
Browse files Browse the repository at this point in the history
  • Loading branch information
Christian Stoeckl committed Jan 20, 2025
1 parent b992e88 commit 3fb801c
Show file tree
Hide file tree
Showing 25 changed files with 615 additions and 118 deletions.
35 changes: 35 additions & 0 deletions jhdf/src/main/java/io/jhdf/BufferBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,23 @@ public BufferBuilder writeInt(int i) {
}
}

public BufferBuilder writeInts(int[] ints) {
int i=0,j=0;
try {
for (j=0; j < ints.length; j++) {
if(BYTE_ORDER == LITTLE_ENDIAN) {
i = Integer.reverseBytes(ints[j]);
} else {
i=ints[j];
}
dataOutputStream.writeInt(i);
}
return this;
} catch (IOException e) {
throw new BufferBuilderException(e);
}
}

public BufferBuilder writeLong(long l) {
try {
if(BYTE_ORDER == LITTLE_ENDIAN) {
Expand All @@ -92,6 +109,24 @@ public BufferBuilder writeLong(long l) {
}
}

public BufferBuilder writeLongs(long[] longs) {
long l=0;
int j=0;
try {
for (j=0; j < longs.length; j++) {
if(BYTE_ORDER == LITTLE_ENDIAN) {
l = Long.reverseBytes(longs[j]);
} else {
l=longs[j];
}
dataOutputStream.writeLong(l);
}
return this;
} catch (IOException e) {
throw new BufferBuilderException(e);
}
}

public ByteBuffer build() {
try {
ByteBuffer byteBuffer = ByteBuffer.wrap(byteArrayOutputStream.toByteArray());
Expand Down
104 changes: 89 additions & 15 deletions jhdf/src/main/java/io/jhdf/ObjectHeader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,25 @@ public static class ObjectHeaderV1 extends ObjectHeader {
*/
private final int referenceCount;

public ObjectHeaderV1(long address, List<Message> messages) {
super(address);
this.messages.addAll(messages);

version = 1;
referenceCount = 0;

// accessTime = -1;
// modificationTime = -1;
// changeTime = -1;
// birthTime = -1;

// maximumNumberOfCompactAttributes = -1;
// maximumNumberOfDenseAttributes = -1;

// flags = new BitSet(8); // TODO make consistent with values
// flags.set(1); // Make sizeOfChunk0 4 bytes.
}

private ObjectHeaderV1(HdfBackingStorage hdfBackingStorage, long address) {
super(address);

Expand Down Expand Up @@ -136,7 +155,8 @@ private void readMessages(HdfBackingStorage hdfBackingStorage, ByteBuffer bb, in
if (m instanceof ObjectHeaderContinuationMessage) {
ObjectHeaderContinuationMessage ohcm = (ObjectHeaderContinuationMessage) m;

ByteBuffer continuationBuffer = hdfBackingStorage.readBufferFromAddress(ohcm.getOffset(), ohcm.getLength());
ByteBuffer continuationBuffer = hdfBackingStorage.readBufferFromAddress(ohcm.getOffset(),
ohcm.getLength());

readMessages(hdfBackingStorage, continuationBuffer, numberOfMessages);
}
Expand All @@ -162,6 +182,46 @@ public boolean isAttributeCreationOrderIndexed() {
return false; // Not supported in v1 headers
}

public ByteBuffer toBuffer() {

// Start messages
ByteBuffer messagesBuffer = messagesToBuffer();

// finish buffer
BufferBuilder bufferBuilder = new BufferBuilder()
.writeByte(version)
.writeByte(0) // reserved
.writeShort(messages.size())
.writeInt(1) // obj. reference count
.writeInt(messagesBuffer.capacity())
.writeInt(0) // reseved
.writeBuffer(messagesBuffer);

return bufferBuilder.build();
}

private ByteBuffer messagesToBuffer() {
BufferBuilder bufferBuilder = new BufferBuilder();
for (Message message : messages) {
final ByteBuffer messageBuffer = message.toBuffer();
int length = messageBuffer.capacity();
if (message.getMessageType() != 1) {
length = length + (8 - (length % 8)); // extend to next 8 byte boundary
}
bufferBuilder.writeShort(message.getMessageType())
.writeShort(length)
.writeBytes(message.flagsToBytes())
.writeByte(0) // padding
.writeByte(0) // padding
.writeByte(0) // padding
.writeBuffer(messageBuffer);
for (int i=messageBuffer.capacity(); i<length; i++) { // extend to next 8 byte boundary
bufferBuilder.writeByte(0); // padding
}
}
return bufferBuilder.build();
}

}

/**
Expand All @@ -173,7 +233,8 @@ public boolean isAttributeCreationOrderIndexed() {
public static class ObjectHeaderV2 extends ObjectHeader {

private static final byte[] OBJECT_HEADER_V2_SIGNATURE = "OHDR".getBytes(StandardCharsets.US_ASCII);
private static final byte[] OBJECT_HEADER_V2_CONTINUATION_SIGNATURE = "OCHK".getBytes(StandardCharsets.US_ASCII);
private static final byte[] OBJECT_HEADER_V2_CONTINUATION_SIGNATURE = "OCHK"
.getBytes(StandardCharsets.US_ASCII);

private static final int ATTRIBUTE_CREATION_ORDER_TRACKED = 2;
private static final int ATTRIBUTE_CREATION_ORDER_INDEXED = 3;
Expand Down Expand Up @@ -219,7 +280,7 @@ private ObjectHeaderV2(HdfBackingStorage hdfBackingStorage, long address) {
}

// Flags
flags = BitSet.valueOf(new byte[]{bb.get()});
flags = BitSet.valueOf(new byte[] { bb.get() });

// Size of chunk 0
final byte sizeOfChunk0 = getSizeOfChunk0Size();
Expand Down Expand Up @@ -316,9 +377,9 @@ public ObjectHeaderV2(long address, List<Message> messages) {

public ByteBuffer toBuffer() {
BufferBuilder bufferBuilder = new BufferBuilder()
.writeBytes(OBJECT_HEADER_V2_SIGNATURE)
.writeByte(version)
.writeBitSet(flags, 1);
.writeBytes(OBJECT_HEADER_V2_SIGNATURE)
.writeByte(version)
.writeBitSet(flags, 1);

if (flags.get(TIMESTAMPS_PRESENT)) {
bufferBuilder.writeInt((int) accessTime);
Expand All @@ -327,7 +388,7 @@ public ByteBuffer toBuffer() {
bufferBuilder.writeInt((int) birthTime);
}

if(flags.get(NUMBER_OF_ATTRIBUTES_PRESENT)) {
if (flags.get(NUMBER_OF_ATTRIBUTES_PRESENT)) {
// TODO min/max attributes
throw new UnsupportedHdfException("Writing number of attributes");
}
Expand All @@ -344,29 +405,41 @@ private ByteBuffer messagesToBuffer() {
BufferBuilder bufferBuilder = new BufferBuilder();
for (Message message : messages) {
final ByteBuffer messageBuffer = message.toBuffer();
bufferBuilder.writeByte(message.getMessageType())
.writeShort(messageBuffer.capacity())
.writeBytes(message.flagsToBytes())
.writeBuffer(messageBuffer);
if (message.getMessageType() == 8) {
bufferBuilder.writeByte(message.getMessageType())
.writeShort(messageBuffer.capacity())
.writeBytes(message.flagsToBytes())
// .writeByte(0) // padding
// .writeByte(0) // padding
// .writeByte(0) // padding
.writeBuffer(messageBuffer);
} else {
bufferBuilder.writeByte(message.getMessageType())
.writeShort(messageBuffer.capacity())
.writeBytes(message.flagsToBytes())
.writeBuffer(messageBuffer);
}
}
return bufferBuilder.build();
}

private void readMessages(HdfBackingStorage hdfBackingStorage, ByteBuffer bb) {
while (bb.remaining() >= 8) {
Message m = Message.readObjectHeaderV2Message(bb, hdfBackingStorage, this.isAttributeCreationOrderTracked());
Message m = Message.readObjectHeaderV2Message(bb, hdfBackingStorage,
this.isAttributeCreationOrderTracked());
messages.add(m);

if (m instanceof ObjectHeaderContinuationMessage) {
ObjectHeaderContinuationMessage ohcm = (ObjectHeaderContinuationMessage) m;
ByteBuffer continuationBuffer = hdfBackingStorage.readBufferFromAddress(ohcm.getOffset(), ohcm.getLength());
ByteBuffer continuationBuffer = hdfBackingStorage.readBufferFromAddress(ohcm.getOffset(),
ohcm.getLength());

// Verify continuation block signature
byte[] continuationSignatureBytes = new byte[OBJECT_HEADER_V2_CONTINUATION_SIGNATURE.length];
continuationBuffer.get(continuationSignatureBytes);
if (!Arrays.equals(OBJECT_HEADER_V2_CONTINUATION_SIGNATURE, continuationSignatureBytes)) {
throw new HdfException(
"Object header continuation header not matched, at address: " + ohcm.getOffset());
"Object header continuation header not matched, at address: " + ohcm.getOffset());
}

// Recursively read messages
Expand Down Expand Up @@ -429,7 +502,8 @@ public static ObjectHeader readObjectHeader(HdfBackingStorage hdfBackingStorage,
}
}

public static LazyInitializer<ObjectHeader> lazyReadObjectHeader(HdfBackingStorage hdfBackingStorage, long address) {
public static LazyInitializer<ObjectHeader> lazyReadObjectHeader(HdfBackingStorage hdfBackingStorage,
long address) {
logger.debug("Creating lazy object header at address: {}", address);
return new LazyInitializer<ObjectHeader>() {

Expand Down
52 changes: 51 additions & 1 deletion jhdf/src/main/java/io/jhdf/Superblock.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,28 @@ public static class SuperblockV0V1 extends Superblock {
private final long addressOfGlobalFreeSpaceIndex;
private final long endOfFileAddress;
private final long driverInformationBlockAddress;
private final long rootLinkNameAddress = 0L;
private final long rootGroupSymbolTableAddress;
public static final long ROOT_GROUP_ADDRESS = 0x60;


public SuperblockV0V1() {
versionOfSuperblock = 0;
versionNumberOfTheFileFreeSpaceInformation = 0;
versionOfRootGroupSymbolTableEntry = 0;
versionOfSharedHeaderMessageFormat = 0;
sizeOfOffsets = 8;
sizeOfLengths = 8;
groupLeafNodeK = 4;
groupInternalNodeK = 16;
baseAddressByte = 0L;
addressOfGlobalFreeSpaceIndex = Constants.UNDEFINED_ADDRESS;
endOfFileAddress = Constants.UNDEFINED_ADDRESS;
driverInformationBlockAddress = Constants.UNDEFINED_ADDRESS;

rootGroupSymbolTableAddress = ROOT_GROUP_ADDRESS;
}


private SuperblockV0V1(FileChannel fc, long address) {
try {
Expand Down Expand Up @@ -426,6 +447,35 @@ public long getDriverInformationBlockAddress() {
public long getRootGroupSymbolTableAddress() {
return rootGroupSymbolTableAddress;
}

public ByteBuffer toBuffer(long endOfFileAddress) {

BufferBuilder bufferBuilder = new BufferBuilder()
.writeBytes(HDF5_FILE_SIGNATURE)
.writeByte(versionOfSuperblock)
.writeByte(versionNumberOfTheFileFreeSpaceInformation)
.writeByte(versionOfRootGroupSymbolTableEntry)
.writeByte(0) //Reserved
.writeByte(versionOfSharedHeaderMessageFormat)
.writeByte(sizeOfOffsets)
.writeByte(sizeOfLengths)
.writeByte(0) //Reserved
.writeShort(groupLeafNodeK)
.writeShort(groupInternalNodeK)
.writeInt(0) //Flags
.writeLong(baseAddressByte)
.writeLong(addressOfGlobalFreeSpaceIndex)
.writeLong(endOfFileAddress)
.writeLong(driverInformationBlockAddress)
.writeLong(rootLinkNameAddress)
.writeLong(rootGroupSymbolTableAddress)
.writeInt(0) //Cache type
.writeInt(0) //Reserved
.writeLong(0L) //Scratch
.writeLong(0L); //Scratch

return bufferBuilder.build();
}
}

public static class SuperblockV2V3 extends Superblock {
Expand All @@ -446,7 +496,7 @@ public SuperblockV2V3() {
sizeOfLengths = 8;
baseAddressByte = 0;
superblockExtensionAddress = Constants.UNDEFINED_ADDRESS;
endOfFileAddress = 500; // TODO
endOfFileAddress = 1000; // TODO
rootGroupObjectHeaderAddress = WritableHdfFile.ROOT_GROUP_ADDRESS;
}

Expand Down
22 changes: 11 additions & 11 deletions jhdf/src/main/java/io/jhdf/Utils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -403,6 +403,17 @@ public static int[] getDimensions(Object data) {
return ArrayUtils.toPrimitive(dims.toArray(new Integer[0]));
}

public static int getDimensionCount(Object data) {
int dimensionCount = 0;
int dimLength = Array.getLength(data);

while (dimLength > 0 && Array.get(data, 0).getClass().isArray()) {
data = Array.get(data, 0);
dimensionCount++;
}
return dimensionCount;
}

public static Class<?> getType(Object obj) {
final Class<?> type;
if(obj.getClass().isArray()) {
Expand Down Expand Up @@ -456,15 +467,4 @@ private static void flattenInternal(Object data, List<Object> flat) {
flat.add(data);
}
}

public static int totalChunks(int[] datasetDimensions, int[] chunkDimensions) {
int chunks = 1;
for (int i = 0; i < datasetDimensions.length; i++) {
int chunksInDim = datasetDimensions[i] / chunkDimensions[i];
// If there is a partial chunk then we need to add one chunk in this dim
if(datasetDimensions[i] % chunkDimensions[i] != 0 ) chunksInDim++;
chunks *= chunksInDim;
}
return chunks;
}
}
Loading

0 comments on commit 3fb801c

Please sign in to comment.