Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Chunked writing + compression proof of concept #673

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Chunked writing + compression proof of concept #673

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions jhdf/src/main/java/io/jhdf/BufferBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,13 @@ public BufferBuilder writeInt(int i) {
throw new BufferBuilderException(e);
}
}

public BufferBuilder writeInts(int[] ints) {
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Think this csn be simplified

	public BufferBuilder writeInts(int[] ints) {
		for (int i=0; i < ints.length; i++) {
			writeInt(ints[i]);
		}
		return this;
	}

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks !

Fixed.

for (int i=0; i < ints.length; i++) {
writeInt(ints[i]);
}
return this;
}

public BufferBuilder writeLong(long l) {
try {
Expand All @@ -91,6 +98,13 @@ public BufferBuilder writeLong(long l) {
throw new BufferBuilderException(e);
}
}

public BufferBuilder writeLongs(long[] longs) {
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

see above

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks !

Fixed.

for (int i=0; i < longs.length; i++) {
writeLong(longs[i]);
}
return this;
}

public ByteBuffer build() {
try {
Expand Down
76 changes: 68 additions & 8 deletions jhdf/src/main/java/io/jhdf/ObjectHeader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,25 @@ public static class ObjectHeaderV1 extends ObjectHeader {
*/
private final int referenceCount;

public ObjectHeaderV1(long address, List<Message> messages) {
super(address);
this.messages.addAll(messages);

version = 1;
referenceCount = 0;

// accessTime = -1;
// modificationTime = -1;
// changeTime = -1;
// birthTime = -1;

// maximumNumberOfCompactAttributes = -1;
// maximumNumberOfDenseAttributes = -1;

// flags = new BitSet(8); // TODO make consistent with values
// flags.set(1); // Make sizeOfChunk0 4 bytes.
}

private ObjectHeaderV1(HdfBackingStorage hdfBackingStorage, long address) {
super(address);

Expand Down Expand Up @@ -136,7 +155,8 @@ private void readMessages(HdfBackingStorage hdfBackingStorage, ByteBuffer bb, in
if (m instanceof ObjectHeaderContinuationMessage) {
ObjectHeaderContinuationMessage ohcm = (ObjectHeaderContinuationMessage) m;

ByteBuffer continuationBuffer = hdfBackingStorage.readBufferFromAddress(ohcm.getOffset(), ohcm.getLength());
ByteBuffer continuationBuffer = hdfBackingStorage.readBufferFromAddress(ohcm.getOffset(),
ohcm.getLength());

readMessages(hdfBackingStorage, continuationBuffer, numberOfMessages);
}
Expand All @@ -162,6 +182,46 @@ public boolean isAttributeCreationOrderIndexed() {
return false; // Not supported in v1 headers
}

public ByteBuffer toBuffer() {

// Start messages
ByteBuffer messagesBuffer = messagesToBuffer();

// finish buffer
BufferBuilder bufferBuilder = new BufferBuilder()
.writeByte(version)
.writeByte(0) // reserved
.writeShort(messages.size())
.writeInt(1) // obj. reference count
.writeInt(messagesBuffer.capacity())
.writeInt(0) // reseved
.writeBuffer(messagesBuffer);

return bufferBuilder.build();
}

private ByteBuffer messagesToBuffer() {
BufferBuilder bufferBuilder = new BufferBuilder();
for (Message message : messages) {
final ByteBuffer messageBuffer = message.toBuffer();
int length = messageBuffer.capacity();
if (message.getMessageType() != 1) {
length = length + (8 - (length % 8)); // extend to next 8 byte boundary
}
bufferBuilder.writeShort(message.getMessageType())
.writeShort(length)
.writeBytes(message.flagsToBytes())
.writeByte(0) // padding
.writeByte(0) // padding
.writeByte(0) // padding
.writeBuffer(messageBuffer);
for (int i=messageBuffer.capacity(); i<length; i++) { // extend to next 8 byte boundary
bufferBuilder.writeByte(0); // padding
}
}
return bufferBuilder.build();
}

}

/**
Expand Down Expand Up @@ -316,9 +376,9 @@ public ObjectHeaderV2(long address, List<Message> messages) {

public ByteBuffer toBuffer() {
BufferBuilder bufferBuilder = new BufferBuilder()
.writeBytes(OBJECT_HEADER_V2_SIGNATURE)
.writeByte(version)
.writeBitSet(flags, 1);
.writeBytes(OBJECT_HEADER_V2_SIGNATURE)
.writeByte(version)
.writeBitSet(flags, 1);

if (flags.get(TIMESTAMPS_PRESENT)) {
bufferBuilder.writeInt((int) accessTime);
Expand All @@ -327,7 +387,7 @@ public ByteBuffer toBuffer() {
bufferBuilder.writeInt((int) birthTime);
}

if(flags.get(NUMBER_OF_ATTRIBUTES_PRESENT)) {
if (flags.get(NUMBER_OF_ATTRIBUTES_PRESENT)) {
// TODO min/max attributes
throw new UnsupportedHdfException("Writing number of attributes");
}
Expand All @@ -345,9 +405,9 @@ private ByteBuffer messagesToBuffer() {
for (Message message : messages) {
final ByteBuffer messageBuffer = message.toBuffer();
bufferBuilder.writeByte(message.getMessageType())
.writeShort(messageBuffer.capacity())
.writeBytes(message.flagsToBytes())
.writeBuffer(messageBuffer);
.writeShort(messageBuffer.capacity())
.writeBytes(message.flagsToBytes())
.writeBuffer(messageBuffer);
}
return bufferBuilder.build();
}
Expand Down
52 changes: 51 additions & 1 deletion jhdf/src/main/java/io/jhdf/Superblock.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,28 @@ public static class SuperblockV0V1 extends Superblock {
private final long addressOfGlobalFreeSpaceIndex;
private final long endOfFileAddress;
private final long driverInformationBlockAddress;
private final long rootLinkNameAddress = 0L;
private final long rootGroupSymbolTableAddress;
public static final long ROOT_GROUP_ADDRESS = 0x60;


public SuperblockV0V1() {
versionOfSuperblock = 0;
versionNumberOfTheFileFreeSpaceInformation = 0;
versionOfRootGroupSymbolTableEntry = 0;
versionOfSharedHeaderMessageFormat = 0;
sizeOfOffsets = 8;
sizeOfLengths = 8;
groupLeafNodeK = 4;
groupInternalNodeK = 16;
baseAddressByte = 0L;
addressOfGlobalFreeSpaceIndex = Constants.UNDEFINED_ADDRESS;
endOfFileAddress = Constants.UNDEFINED_ADDRESS;
driverInformationBlockAddress = Constants.UNDEFINED_ADDRESS;

rootGroupSymbolTableAddress = ROOT_GROUP_ADDRESS;
}


private SuperblockV0V1(FileChannel fc, long address) {
try {
Expand Down Expand Up @@ -426,6 +447,35 @@ public long getDriverInformationBlockAddress() {
public long getRootGroupSymbolTableAddress() {
return rootGroupSymbolTableAddress;
}

public ByteBuffer toBuffer(long endOfFileAddress) {

BufferBuilder bufferBuilder = new BufferBuilder()
.writeBytes(HDF5_FILE_SIGNATURE)
.writeByte(versionOfSuperblock)
.writeByte(versionNumberOfTheFileFreeSpaceInformation)
.writeByte(versionOfRootGroupSymbolTableEntry)
.writeByte(0) //Reserved
.writeByte(versionOfSharedHeaderMessageFormat)
.writeByte(sizeOfOffsets)
.writeByte(sizeOfLengths)
.writeByte(0) //Reserved
.writeShort(groupLeafNodeK)
.writeShort(groupInternalNodeK)
.writeInt(0) //Flags
.writeLong(baseAddressByte)
.writeLong(addressOfGlobalFreeSpaceIndex)
.writeLong(endOfFileAddress)
.writeLong(driverInformationBlockAddress)
.writeLong(rootLinkNameAddress)
.writeLong(rootGroupSymbolTableAddress)
.writeInt(0) //Cache type
.writeInt(0) //Reserved
.writeLong(0L) //Scratch
.writeLong(0L); //Scratch

return bufferBuilder.build();
}
}

public static class SuperblockV2V3 extends Superblock {
Expand All @@ -446,7 +496,7 @@ public SuperblockV2V3() {
sizeOfLengths = 8;
baseAddressByte = 0;
superblockExtensionAddress = Constants.UNDEFINED_ADDRESS;
endOfFileAddress = 500; // TODO
endOfFileAddress = 1000; // TODO
rootGroupObjectHeaderAddress = WritableHdfFile.ROOT_GROUP_ADDRESS;
}

Expand Down
Loading
Loading