diff --git a/README.md b/README.md index 56f7583..86dea31 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,10 @@ way to store and retrieve data from RocksDB. - [x] **MultiSet (RocksMultiSet) :** A basic implementation for set where you can add, remove and check if an item exists. It supports multiple sets under the same name, but different namespaces. - [x] **ZSet (RocksZSet) :** A basic implementation for sorted set where you can add, remove and check if an item exists. You can find score of an item and items within a score range. - [x] **MultiZSet (RocksMultiZSet) :** A basic implementation for sorted set where you can add, remove and check if an item exists. You can find score of an item and items within a score range. It supports multiple sorted sets under the same name, but different namespaces. -- [ ] **Map (RocksMap) :** Not implemented yet +- [x] **Map (RocksMap) :** A basic implementation for map where you can add, remove and check if an item exists. +- [x] **MultiMap (RocksMultiMap) :** A basic implementation for map where you can add, remove and check if an item exists. It supports multiple maps under the same name, but different namespaces. +- [x] **Bitmap (RocksBitmap) :** A basic implementation for bitmap where you can set, unset and check if a bit is set. +- [x] **MultiBitmap (RocksMultiBitmap) :** A basic implementation for bitmap where you can set, unset and check if a bit is set. It supports multiple bitmaps under the same name, but different namespaces. ## Pre-requisites diff --git a/build.gradle b/build.gradle index c062c20..f3b1024 100644 --- a/build.gradle +++ b/build.gradle @@ -105,6 +105,7 @@ subprojects { maven { url "https://oss.sonatype.org/content/repositories/snapshots" } + maven { url 'https://jitpack.io' } } archivesBaseName = 'rocks-types-' + project.name diff --git a/core/build.gradle b/core/build.gradle index 75b2d51..f7c59b3 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -2,6 +2,7 @@ dependencies { api libs.slf4j.api api libs.rocksdb + api libs.roaringbitmap implementation libs.messagepack.jackson } diff --git a/core/src/main/java/com/bloxbean/rocks/types/collection/BaseDataType.java b/core/src/main/java/com/bloxbean/rocks/types/collection/BaseDataType.java index edc4f45..87ffe31 100644 --- a/core/src/main/java/com/bloxbean/rocks/types/collection/BaseDataType.java +++ b/core/src/main/java/com/bloxbean/rocks/types/collection/BaseDataType.java @@ -57,6 +57,24 @@ protected void writeBatch(WriteBatch batch, byte[] key, byte[] value) { } } + @SneakyThrows + protected void merge(WriteBatch writeBatch, byte[] key, byte[] value) { + if (writeBatch != null) { + mergeBatch(writeBatch, key, value); + } else { + merge(key, value); + } + } + + @SneakyThrows + protected void mergeBatch(WriteBatch batch, byte[] key, byte[] value) { + if (columnFamilyHandle != null) { + batch.merge(columnFamilyHandle, key, value); + } else { + batch.merge(key, value); + } + } + @SneakyThrows protected void deleteBatch(WriteBatch batch, byte[] key) { if (columnFamilyHandle != null) { @@ -93,6 +111,15 @@ protected void delete(byte[] key) { } } + @SneakyThrows + protected void merge(byte[] key, byte[] value) { + if (columnFamilyHandle != null) { + db.merge(columnFamilyHandle, key, value); + } else { + db.merge(key, value); + } + } + protected RocksIterator iterator() { if (columnFamilyHandle != null) { return db.newIterator(columnFamilyHandle); diff --git a/core/src/main/java/com/bloxbean/rocks/types/collection/RocksBitmap.java b/core/src/main/java/com/bloxbean/rocks/types/collection/RocksBitmap.java new file mode 100644 index 0000000..3831a82 --- /dev/null +++ b/core/src/main/java/com/bloxbean/rocks/types/collection/RocksBitmap.java @@ -0,0 +1,72 @@ +package com.bloxbean.rocks.types.collection; + +import com.bloxbean.rocks.types.config.RocksDBConfig; +import org.roaringbitmap.RoaringBitmap; +import org.rocksdb.WriteBatch; + +/** + * Bitmap implementation using RocksDB + * RocksBitmap is a wrapper class for RocksMultiBitmap with default column family + */ +public class RocksBitmap extends RocksMultiBitmap { + + public RocksBitmap(RocksDBConfig rocksDBConfig, String name) { + super(rocksDBConfig, name); + } + + public RocksBitmap(RocksDBConfig rocksDBConfig, String name, int fragmentSize) { + super(rocksDBConfig, name, fragmentSize); + } + + public RocksBitmap(RocksDBConfig rocksDBConfig, String columnFamily, String name) { + super(rocksDBConfig, columnFamily, name); + } + + public RocksBitmap(RocksDBConfig rocksDBConfig, String columnFamily, String name, int fragmentSize) { + super(rocksDBConfig, columnFamily, name, fragmentSize); + } + + public void setBit(int index) { + super.setBit(null, index); + } + + public void setBitBatch(WriteBatch writeBatch, int... bitIndexes) { + super.setBitBatch(null, writeBatch, bitIndexes); + } + + public boolean getBit(int bitIndex) { + return super.getBit(null, bitIndex); + } + + public void clearBit(int bitIndex) { + super.clearBit(null, bitIndex); + } + + public void clearBitBatch(WriteBatch writeBatch, int... bitIndexes) { + super.clearBitBatch(null, writeBatch, bitIndexes); + } + + public long nextSetBit(int fromIndex) { + return super.nextSetBit(null, fromIndex); + } + + public long nextClearBit(int fromIndex) { + return super.nextClearBit(null, fromIndex); + } + + public long previousSetBit(int fromIndex) { + return super.previousSetBit(null, fromIndex); + } + + public long previousClearBit(int fromIndex) { + return super.previousClearBit(null, fromIndex); + } + + public RoaringBitmap getAllBits() { + return super.getAllBits(null); + } + + public RoaringBitmap getBits(int fromFragmentIndex, int toFragmentIndex) { + return super.getBits(null, fromFragmentIndex, toFragmentIndex); + } +} diff --git a/core/src/main/java/com/bloxbean/rocks/types/collection/RocksMultiBitmap.java b/core/src/main/java/com/bloxbean/rocks/types/collection/RocksMultiBitmap.java new file mode 100644 index 0000000..53d1a68 --- /dev/null +++ b/core/src/main/java/com/bloxbean/rocks/types/collection/RocksMultiBitmap.java @@ -0,0 +1,440 @@ +package com.bloxbean.rocks.types.collection; + +import com.bloxbean.rocks.types.collection.metadata.BitmapMetadata; +import com.bloxbean.rocks.types.common.KeyBuilder; +import com.bloxbean.rocks.types.config.RocksDBConfig; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.roaringbitmap.RoaringBitmap; +import org.rocksdb.WriteBatch; + +import java.io.*; +import java.util.Optional; + +/** + * Bitmap implementation using RocksDB. It supports multiple bitmaps under the same name but different namespaces. + * This implementation uses RoaringBitmap as the underlying bitmap implementation. + *

+ * This implementation is not thread safe. + */ +@Slf4j +public class RocksMultiBitmap extends BaseDataType { + private static final int DEFAULT_FRAGMENT_SIZE = 1000; //1000 bits + private int fragmentSize; + + public RocksMultiBitmap(RocksDBConfig rocksDBConfig, String name) { + super(rocksDBConfig, name, null); + this.fragmentSize = DEFAULT_FRAGMENT_SIZE; + } + + public RocksMultiBitmap(RocksDBConfig rocksDBConfig, String name, int fragmentSize) { + super(rocksDBConfig, name, null); + this.fragmentSize = fragmentSize; + } + + public RocksMultiBitmap(RocksDBConfig rocksDBConfig, String columnFamily, String name) { + super(rocksDBConfig, columnFamily, name, null); + } + + public RocksMultiBitmap(RocksDBConfig rocksDBConfig, String columnFamily, String name, int fragmentSize) { + super(rocksDBConfig, columnFamily, name, null); + this.fragmentSize = fragmentSize; + } + + public void setBit(String ns, int bitIndex) { + var metadata = createMetadata(ns).orElseThrow(); + setBit(ns, null, metadata, bitIndex); + } + + public void setBitBatch(String ns, WriteBatch writeBatch, int... bitIndexes) { + var metadata = createMetadata(ns).orElseThrow(); + for (int bitIndex : bitIndexes) { + setBit(ns, writeBatch, metadata, bitIndex); + } + } + + @SneakyThrows + private void setBit(String ns, WriteBatch writeBatch, BitmapMetadata metadata, int bitIndex) { + + int fragmentIndex = bitIndex / fragmentSize; + int fragmentBitIndex = bitIndex % fragmentSize; + + byte[] keyBytes = getKey(metadata, ns, fragmentIndex); + + + RoaringBitmap bitSet = null; + byte[] valueBytes = get(keyBytes); + if (valueBytes == null || valueBytes.length == 0) { + bitSet = new RoaringBitmap(); + } else { + bitSet = getRoaringBitmap(valueBytes); + } + + bitSet.add(fragmentBitIndex); + + byte[] bytes = serializeRoaringBitmap(bitSet); + + write(writeBatch, keyBytes, bytes); + if (fragmentIndex > metadata.getMaxFragmentIndex()) { + metadata.setMaxFragmentIndex(fragmentIndex); + updateMetadata(writeBatch, metadata, ns); + log.info("Max fragment index updated to {}", metadata); + } + } + + private static byte[] serializeRoaringBitmap(RoaringBitmap bitSet) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + bitSet.serialize(new DataOutputStream(baos)); + return baos.toByteArray(); + } + + private static RoaringBitmap getRoaringBitmap(byte[] valueBytes) throws IOException { + RoaringBitmap bitSet; + bitSet = new RoaringBitmap();//BitSet.valueOf(valueBytes); + ByteArrayInputStream bais = new ByteArrayInputStream(valueBytes); + bitSet.deserialize(new DataInputStream(bais)); + return bitSet; + } + + public void clearBit(String ns, int bitIndex) { + var metadata = createMetadata(ns).orElseThrow(); + clearBit(ns, null, metadata, bitIndex); + } + + public void clearBitBatch(String ns, WriteBatch writeBatch, int... bitIndexes) { + var metadata = createMetadata(ns).orElseThrow(); + for (int bitIndex : bitIndexes) { + clearBit(ns, writeBatch, metadata, bitIndex); + } + } + + @SneakyThrows + private void clearBit(String ns, WriteBatch writeBatch, BitmapMetadata metadata, int bitIndex) { + + int fragmentIndex = bitIndex / fragmentSize; + int fragmentBitIndex = bitIndex % fragmentSize; + + byte[] keyBytes = getKey(metadata, ns, fragmentIndex); + + byte[] valueBytes = get(keyBytes); + if (valueBytes == null || valueBytes.length == 0) { + return; //nothing to clear + } + + var bitSet = getRoaringBitmap(valueBytes); + bitSet.remove(fragmentBitIndex); + + byte[] serializedBytes = serializeRoaringBitmap(bitSet); + + write(writeBatch, keyBytes, serializedBytes); + } + + public boolean getBit(String ns, int bitIndex) { + var metadata = getMetadata(ns).orElseThrow(); + return getBit(ns, metadata, bitIndex); + } + + @SneakyThrows + private boolean getBit(String ns, BitmapMetadata metadata, int bitIndex) { + + int fragmentIndex = bitIndex / fragmentSize; + int fragmentBitIndex = bitIndex % fragmentSize; + + byte[] keyBytes = getKey(metadata, ns, fragmentIndex); + + byte[] valueBytes = get(keyBytes); + if (valueBytes == null || valueBytes.length == 0) { + return false; + } + + var bitSet = getRoaringBitmap(valueBytes); + return bitSet.contains(fragmentBitIndex); + } + + //Get all the bits set in the bitmap in all fragments + public RoaringBitmap getAllBits(String ns) { + var metadata = getMetadata(ns).orElseThrow(); + return getAllBits(ns, metadata); + } + + @SneakyThrows + private RoaringBitmap getAllBits(String ns, BitmapMetadata metadata) { + RoaringBitmap roaringBitmap = null; + for (int i = 0; i <= metadata.getMaxFragmentIndex(); i++) { + byte[] keyBytes = getKey(metadata, ns, i); + byte[] valueBytes = get(keyBytes); + RoaringBitmap fragmentBitSet = null; + if (valueBytes == null || valueBytes.length == 0) { + fragmentBitSet = new RoaringBitmap(); + } else { + fragmentBitSet = getRoaringBitmap(valueBytes); + } + + fragmentBitSet = RoaringBitmap.addOffset(fragmentBitSet, i * fragmentSize); + + if (roaringBitmap == null) { + roaringBitmap = fragmentBitSet; + } else { + roaringBitmap.or(fragmentBitSet); + } + } + + return roaringBitmap; + } + + public RoaringBitmap getBits(String ns, int fromFragmentIndex, int toFragmentIndex) { + var metadata = getMetadata(ns).orElseThrow(); + return getBits(ns, metadata, fromFragmentIndex, toFragmentIndex); + } + + @SneakyThrows + private RoaringBitmap getBits(String ns, BitmapMetadata metadata, int fromFragmentIndex, int toFragmentIndex) { + RoaringBitmap roaringBitmap = null; + for (int i = fromFragmentIndex; i <= toFragmentIndex; i++) { + byte[] keyBytes = getKey(metadata, ns, i); + byte[] valueBytes = get(keyBytes); + RoaringBitmap fragmentBitSet = null; + if (valueBytes == null || valueBytes.length == 0) { + fragmentBitSet = new RoaringBitmap(); + } else { + fragmentBitSet = getRoaringBitmap(valueBytes); + } + + fragmentBitSet = RoaringBitmap.addOffset(fragmentBitSet, i * fragmentSize); + + if (roaringBitmap == null) { + roaringBitmap = fragmentBitSet; + } else { + roaringBitmap.or(fragmentBitSet); + } + } + + return roaringBitmap; + } + + public long nextSetBit(String ns, int fromIndex) { + var metadata = getMetadata(ns).orElseThrow(); + return nextSetBit(ns, metadata, fromIndex); + } + + @SneakyThrows + private long nextSetBit(String ns, BitmapMetadata metadata, int fromIndex) { + int fragmentIndex = fromIndex / fragmentSize; + int fragmentBitIndex = fromIndex % fragmentSize; + + byte[] keyBytes = getKey(metadata, ns, fragmentIndex); + + byte[] valueBytes = get(keyBytes); + RoaringBitmap bitSet = null; + if (valueBytes == null || valueBytes.length == 0) { + bitSet = new RoaringBitmap(); + } else { + bitSet = getRoaringBitmap(valueBytes); + } + + long nextSetBit = bitSet.nextValue(fragmentBitIndex); + if (nextSetBit == -1) { + for (int i = fragmentIndex + 1; i <= metadata.getMaxFragmentIndex(); i++) { + byte[] nextKeyBytes = getKey(metadata, ns, i); + byte[] nextValueBytes = get(nextKeyBytes); + if (nextValueBytes == null || nextValueBytes.length == 0) { + continue; + } + + var nextBitSet = getRoaringBitmap(nextValueBytes); + nextSetBit = nextBitSet.nextValue(0); + if (nextSetBit != -1) { + return nextSetBit + (i * fragmentSize); + } + } + } else { + return nextSetBit + (fragmentIndex * fragmentSize); + } + + return -1; + } + + public long nextClearBit(String ns, int fromIndex) { + var metadata = getMetadata(ns).orElseThrow(); + return nextClearBit(ns, metadata, fromIndex); + } + + @SneakyThrows + private long nextClearBit(String ns, BitmapMetadata metadata, int fromIndex) { + int fragmentIndex = fromIndex / fragmentSize; + int fragmentBitIndex = fromIndex % fragmentSize; + + byte[] keyBytes = getKey(metadata, ns, fragmentIndex); + + RoaringBitmap bitSet = null; + byte[] valueBytes = get(keyBytes); + if (valueBytes == null || valueBytes.length == 0) { + bitSet = new RoaringBitmap(); + } else { + bitSet = getRoaringBitmap(valueBytes); + } + + long nextClearBit = bitSet.nextAbsentValue(fragmentBitIndex); + if (nextClearBit == -1) { + for (int i = fragmentIndex + 1; i <= metadata.getMaxFragmentIndex(); i++) { + byte[] nextKeyBytes = getKey(metadata, ns, i); + byte[] nextValueBytes = get(nextKeyBytes); + if (nextValueBytes == null || nextValueBytes.length == 0) { + continue; + } + + var nextBitSet = getRoaringBitmap(nextValueBytes); + nextClearBit = nextBitSet.nextAbsentValue(0); + if (nextClearBit != -1) { + return nextClearBit + (i * fragmentSize); + } + } + } else { + return nextClearBit + (fragmentIndex * fragmentSize); + } + + return -1; + } + + public long previousSetBit(String ns, int fromIndex) { + var metadata = getMetadata(ns).orElseThrow(); + return previousSetBit(ns, metadata, fromIndex); + } + + @SneakyThrows + private long previousSetBit(String ns, BitmapMetadata metadata, int fromIndex) { + int fragmentIndex = fromIndex / fragmentSize; + int fragmentBitIndex = fromIndex % fragmentSize; + + byte[] keyBytes = getKey(metadata, ns, fragmentIndex); + + RoaringBitmap bitSet = null; + byte[] valueBytes = get(keyBytes); + if (valueBytes == null || valueBytes.length == 0) { + bitSet = new RoaringBitmap(); + } else { + bitSet = getRoaringBitmap(valueBytes); + } + + long previousSetBit = bitSet.previousValue(fragmentBitIndex); + if (previousSetBit == -1) { + for (int i = fragmentIndex - 1; i >= 0; i--) { + byte[] nextKeyBytes = getKey(metadata, ns, i); + byte[] nextValueBytes = get(nextKeyBytes); + RoaringBitmap nextBitSet = null; + if (nextValueBytes == null || nextValueBytes.length == 0) { + nextBitSet = new RoaringBitmap(); + } else { + nextBitSet = getRoaringBitmap(nextValueBytes); + } + + previousSetBit = nextBitSet.previousValue(fragmentSize - 1); + if (previousSetBit != -1) { + return previousSetBit + (i * fragmentSize); + } + } + } else { + return previousSetBit + (fragmentIndex * fragmentSize); + } + + return -1; + } + + public long previousClearBit(String ns, int fromIndex) { + var metadata = getMetadata(ns).orElseThrow(); + return previousClearBit(ns, metadata, fromIndex); + } + + @SneakyThrows + private long previousClearBit(String ns, BitmapMetadata metadata, int fromIndex) { + int fragmentIndex = fromIndex / fragmentSize; + int fragmentBitIndex = fromIndex % fragmentSize; + + byte[] keyBytes = getKey(metadata, ns, fragmentIndex); + + RoaringBitmap bitSet = null; + byte[] valueBytes = get(keyBytes); + if (valueBytes == null || valueBytes.length == 0) { + bitSet = new RoaringBitmap(); + } else { + bitSet = getRoaringBitmap(valueBytes); + } + + long previousClearBit = bitSet.previousAbsentValue(fragmentBitIndex); + if (previousClearBit == -1) { + for (int i = fragmentIndex - 1; i >= 0; i--) { + byte[] nextKeyBytes = getKey(metadata, ns, i); + byte[] nextValueBytes = get(nextKeyBytes); + RoaringBitmap nextBitSet = null; + if (nextValueBytes == null || nextValueBytes.length == 0) { + nextBitSet = new RoaringBitmap(); + } else { + nextBitSet = getRoaringBitmap(nextValueBytes); + } + + previousClearBit = nextBitSet.previousAbsentValue(fragmentSize - 1); + if (previousClearBit != -1) { + return previousClearBit + (i * fragmentSize); + } + } + } else { + return previousClearBit + (fragmentIndex * fragmentSize); + } + + return -1; + } + + @SneakyThrows + private BitmapMetadata updateMetadata(WriteBatch writeBatch, BitmapMetadata metadata, String ns) { + var metadataKeyName = getMetadataKey(ns); + write(writeBatch, metadataKeyName, valueSerializer.serialize(metadata)); + return metadata; + } + + @SneakyThrows + protected Optional getMetadata(String ns) { + byte[] metadataKeyName = getMetadataKey(ns); + var metadataValueBytes = get(metadataKeyName); + if (metadataValueBytes == null || metadataValueBytes.length == 0) { + return Optional.empty(); + } else { + return Optional.of(valueSerializer.deserialize(metadataValueBytes, BitmapMetadata.class)); + } + } + + @Override + protected Optional createMetadata(String ns) { + byte[] metadataKeyName = getMetadataKey(ns); + var metadata = getMetadata(ns); + if (metadata.isEmpty()) { + var newMetadata = new BitmapMetadata(); + newMetadata.setVersion(System.currentTimeMillis()); + write(null, metadataKeyName, valueSerializer.serialize(newMetadata)); + return Optional.of(newMetadata); + } else { + return metadata; + } + } + + protected byte[] getMetadataKey(String ns) { + if (ns != null) + return new KeyBuilder(name, ns) + .build(); + else + return new KeyBuilder(name) + .build(); + } + + private byte[] getKey(BitmapMetadata metadata, String ns, int fragmentIndex) { + if (ns != null) + return new KeyBuilder(name, ns) + .append(metadata.getVersion()) + .append(fragmentIndex) + .build(); + else + return new KeyBuilder(name) + .append(metadata.getVersion()) + .append(fragmentIndex) + .build(); + } +} diff --git a/core/src/main/java/com/bloxbean/rocks/types/collection/metadata/BitmapMetadata.java b/core/src/main/java/com/bloxbean/rocks/types/collection/metadata/BitmapMetadata.java new file mode 100644 index 0000000..6ed9499 --- /dev/null +++ b/core/src/main/java/com/bloxbean/rocks/types/collection/metadata/BitmapMetadata.java @@ -0,0 +1,12 @@ +package com.bloxbean.rocks.types.collection.metadata; + +import lombok.Data; + +@Data +public class BitmapMetadata extends TypeMetadata { + private long maxFragmentIndex; + @Override + public DataType getType() { + return DataType.BITMAP; + } +} diff --git a/core/src/main/java/com/bloxbean/rocks/types/collection/metadata/DataType.java b/core/src/main/java/com/bloxbean/rocks/types/collection/metadata/DataType.java index 087f13f..80d5087 100644 --- a/core/src/main/java/com/bloxbean/rocks/types/collection/metadata/DataType.java +++ b/core/src/main/java/com/bloxbean/rocks/types/collection/metadata/DataType.java @@ -4,7 +4,8 @@ public enum DataType { LIST((short) 0), SET((short) 1), MAP((short) 2), - ZSET((short) 3); + ZSET((short) 3), + BITMAP((short) 4); private final short value; DataType(short value) { diff --git a/core/src/test/java/com/bloxbean/rocks/types/collection/RocksBitmapTest.java b/core/src/test/java/com/bloxbean/rocks/types/collection/RocksBitmapTest.java new file mode 100644 index 0000000..988010f --- /dev/null +++ b/core/src/test/java/com/bloxbean/rocks/types/collection/RocksBitmapTest.java @@ -0,0 +1,311 @@ +package com.bloxbean.rocks.types.collection; + +import org.junit.jupiter.api.Test; +import org.roaringbitmap.RoaringBitmap; +import org.rocksdb.WriteBatch; +import org.rocksdb.WriteOptions; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +class RocksBitmapTest extends RocksBaseTest { + + @Test + void setBit() { + for (int i = 0; i < 100; i++) { + var rocksBitMap = new RocksBitmap(rocksDBConfig, "test-bitmap-" + i, 10 * (i + 1)); + rocksBitMap.setBit(1); + rocksBitMap.setBit(50); + rocksBitMap.setBit(99); + + rocksBitMap.setBit(101); + rocksBitMap.setBit(150); + rocksBitMap.setBit(199); + + rocksBitMap.setBit(201); + rocksBitMap.setBit(250); + + assertThat(rocksBitMap.getBit(1)).isTrue(); + assertThat(rocksBitMap.getBit(50)).isTrue(); + assertThat(rocksBitMap.getBit(99)).isTrue(); + assertThat(rocksBitMap.getBit(101)).isTrue(); + assertThat(rocksBitMap.getBit(150)).isTrue(); + assertThat(rocksBitMap.getBit(199)).isTrue(); + assertThat(rocksBitMap.getBit(201)).isTrue(); + assertThat(rocksBitMap.getBit(250)).isTrue(); + + assertThat(rocksBitMap.getBit(41)).isFalse(); + assertThat(rocksBitMap.getBit(100)).isFalse(); + assertThat(rocksBitMap.getBit(200)).isFalse(); + assertThat(rocksBitMap.getBit(240)).isFalse(); + assertThat(rocksBitMap.getBit(350)).isFalse(); + } + } + + @Test + void setBit_batch() throws Exception { + var rocksBitMap = new RocksBitmap(rocksDBConfig, "test-bitmap", 20); + WriteBatch writeBatch = new WriteBatch(); + rocksBitMap.setBitBatch(writeBatch, 1, 50, 99, 101, 150, 199, 201, 250); + + rocksDBConfig.getRocksDB().write(new WriteOptions(), writeBatch); + + assertThat(rocksBitMap.getBit(1)).isTrue(); + assertThat(rocksBitMap.getBit(50)).isTrue(); + assertThat(rocksBitMap.getBit(99)).isTrue(); + assertThat(rocksBitMap.getBit(101)).isTrue(); + assertThat(rocksBitMap.getBit(150)).isTrue(); + assertThat(rocksBitMap.getBit(199)).isTrue(); + assertThat(rocksBitMap.getBit(201)).isTrue(); + assertThat(rocksBitMap.getBit(250)).isTrue(); + + assertThat(rocksBitMap.getBit(41)).isFalse(); + assertThat(rocksBitMap.getBit(100)).isFalse(); + assertThat(rocksBitMap.getBit(200)).isFalse(); + assertThat(rocksBitMap.getBit(240)).isFalse(); + assertThat(rocksBitMap.getBit(350)).isFalse(); + } + + @Test + void clearBit() { + var rocksBitMap = new RocksBitmap(rocksDBConfig, "test-bitmap"); + rocksBitMap.setBit(1); + rocksBitMap.setBit(50); + rocksBitMap.setBit(99); + + rocksBitMap.setBit(101); + rocksBitMap.setBit(150); + rocksBitMap.setBit(199); + + rocksBitMap.setBit(201); + rocksBitMap.setBit(250); + + rocksBitMap.clearBit(99); + rocksBitMap.clearBit(250); + + assertThat(rocksBitMap.getBit(1)).isTrue(); + assertThat(rocksBitMap.getBit(50)).isTrue(); + assertThat(rocksBitMap.getBit(99)).isFalse(); + assertThat(rocksBitMap.getBit(101)).isTrue(); + assertThat(rocksBitMap.getBit(150)).isTrue(); + assertThat(rocksBitMap.getBit(199)).isTrue(); + assertThat(rocksBitMap.getBit(201)).isTrue(); + assertThat(rocksBitMap.getBit(250)).isFalse(); + + } + + @Test + void clearBit_batch() throws Exception { + var rocksBitMap = new RocksBitmap(rocksDBConfig, "test-bitmap", 30); + rocksBitMap.setBit(1); + rocksBitMap.setBit(50); + rocksBitMap.setBit(99); + + rocksBitMap.setBit(101); + rocksBitMap.setBit(150); + rocksBitMap.setBit(199); + + rocksBitMap.setBit(201); + rocksBitMap.setBit(250); + + var writeBatch = new WriteBatch(); + rocksBitMap.clearBitBatch(writeBatch, 99, 250); + + rocksDBConfig.getRocksDB().write(new WriteOptions(), writeBatch); + + assertThat(rocksBitMap.getBit(1)).isTrue(); + assertThat(rocksBitMap.getBit(50)).isTrue(); + assertThat(rocksBitMap.getBit(99)).isFalse(); + assertThat(rocksBitMap.getBit(101)).isTrue(); + assertThat(rocksBitMap.getBit(150)).isTrue(); + assertThat(rocksBitMap.getBit(199)).isTrue(); + assertThat(rocksBitMap.getBit(201)).isTrue(); + assertThat(rocksBitMap.getBit(250)).isFalse(); + + } + + @Test + void getAllBits() { + var rocksBitMap = new RocksBitmap(rocksDBConfig, "test-bitmap", 30); + rocksBitMap.setBit(1); + rocksBitMap.setBit(50); + rocksBitMap.setBit(99); + + rocksBitMap.setBit(101); + rocksBitMap.setBit(150); + rocksBitMap.setBit(199); + + rocksBitMap.setBit(201); + rocksBitMap.setBit(250); + + var allBits = rocksBitMap.getAllBits(); + + var expectedBitSet = new RoaringBitmap(); + expectedBitSet.add(1); + expectedBitSet.add(50); + expectedBitSet.add(99); + expectedBitSet.add(101); + expectedBitSet.add(150); + expectedBitSet.add(199); + expectedBitSet.add(201); + expectedBitSet.add(250); + + System.out.println(allBits); + + assertThat(allBits).isEqualTo(expectedBitSet); + } + + @Test + void getBits() { + var rocksBitMap = new RocksBitmap(rocksDBConfig, "test-bitmap", 30); + rocksBitMap.setBit(1); + rocksBitMap.setBit(50); + rocksBitMap.setBit(99); + + rocksBitMap.setBit(101); + rocksBitMap.setBit(150); + rocksBitMap.setBit(199); + + rocksBitMap.setBit(201); + rocksBitMap.setBit(250); + + var bits = rocksBitMap.getBits(2, 5); + + var expectedBits = new RoaringBitmap(); + expectedBits.add(99); + expectedBits.add(101); + expectedBits.add(150); + + assertThat(bits).isEqualTo(expectedBits); + } + + @Test + void nextSetBit() { + for (int i = 0; i < 10; i++) { + var rocksBitMap = new RocksBitmap(rocksDBConfig, "test-bitmap-" + i, (10 * (i + 1))); + rocksBitMap.setBit(1); + rocksBitMap.setBit(50); + rocksBitMap.setBit(99); + + rocksBitMap.setBit(101); + rocksBitMap.setBit(150); + rocksBitMap.setBit(199); + + rocksBitMap.setBit(201); + rocksBitMap.setBit(250); + + long index1 = rocksBitMap.nextSetBit(140); + long index2 = rocksBitMap.nextSetBit(199); + long index3 = rocksBitMap.nextSetBit(202); + long index4 = rocksBitMap.nextSetBit(250); + long index5 = rocksBitMap.nextSetBit(251); + + assertThat(index1).isEqualTo(150); + assertThat(index2).isEqualTo(199); + assertThat(index3).isEqualTo(250); + assertThat(index4).isEqualTo(250); + assertThat(index5).isEqualTo(-1); + } + } + + @Test + void nextClearBit() { + for (int i = 0; i < 10; i++) { + System.out.println(i); + var rocksBitMap = new RocksBitmap(rocksDBConfig, "test-bitmap-" + i, 10 * (i + 1)); + rocksBitMap.setBit(1); + rocksBitMap.setBit(50); + rocksBitMap.setBit(99); + + rocksBitMap.setBit(101); + rocksBitMap.setBit(150); + rocksBitMap.setBit(199); + + rocksBitMap.setBit(201); + rocksBitMap.setBit(250); + + long index1 = rocksBitMap.nextClearBit(140); + long index2 = rocksBitMap.nextClearBit(199); + long index3 = rocksBitMap.nextClearBit(202); + long index4 = rocksBitMap.nextClearBit(250); + long index5 = rocksBitMap.nextClearBit(251); + + assertThat(index1).isEqualTo(140); + assertThat(index2).isEqualTo(200); + assertThat(index3).isEqualTo(202); + assertThat(index4).isEqualTo(251); + assertThat(index5).isEqualTo(251); + } + } + + @Test + void prevSetBit() { + for (int i = 0; i < 10; i++) { + var rocksBitMap = new RocksBitmap(rocksDBConfig, "test-bitmap-" + i, 10 * (i + 1)); + rocksBitMap.setBit(1); + rocksBitMap.setBit(50); + rocksBitMap.setBit(99); + + rocksBitMap.setBit(101); + rocksBitMap.setBit(150); + rocksBitMap.setBit(199); + + rocksBitMap.setBit(201); + rocksBitMap.setBit(250); + + long index1 = rocksBitMap.previousSetBit(140); + long index2 = rocksBitMap.previousSetBit(199); + long index3 = rocksBitMap.previousSetBit(202); + long index4 = rocksBitMap.previousSetBit(250); + long index5 = rocksBitMap.previousSetBit(251); + long index6 = rocksBitMap.previousSetBit(49); + long index7 = rocksBitMap.previousSetBit(1); + long index8 = rocksBitMap.previousSetBit(0); + + assertThat(index1).isEqualTo(101); + assertThat(index2).isEqualTo(199); + assertThat(index3).isEqualTo(201); + assertThat(index4).isEqualTo(250); + assertThat(index5).isEqualTo(250); + assertThat(index6).isEqualTo(1); + assertThat(index7).isEqualTo(1); + assertThat(index8).isEqualTo(-1); + } + } + + @Test + void prevClearBit() { + for (int i = 0; i < 10; i++) { + var rocksBitMap = new RocksBitmap(rocksDBConfig, "test-bitmap-" + i, 10 * (i + 1)); + rocksBitMap.setBit(1); + rocksBitMap.setBit(50); + rocksBitMap.setBit(99); + + rocksBitMap.setBit(101); + rocksBitMap.setBit(150); + rocksBitMap.setBit(199); + + rocksBitMap.setBit(201); + rocksBitMap.setBit(250); + + System.out.println(i); + + long index1 = rocksBitMap.previousClearBit(140); + long index2 = rocksBitMap.previousClearBit(199); + long index3 = rocksBitMap.previousClearBit(202); + long index4 = rocksBitMap.previousClearBit(250); + long index5 = rocksBitMap.previousClearBit(251); + long index6 = rocksBitMap.previousClearBit(49); + long index7 = rocksBitMap.previousClearBit(1); + long index8 = rocksBitMap.previousClearBit(0); + + assertThat(index1).isEqualTo(140); + assertThat(index2).isEqualTo(198); + assertThat(index3).isEqualTo(202); + assertThat(index4).isEqualTo(249); + assertThat(index5).isEqualTo(251); + assertThat(index6).isEqualTo(49); + assertThat(index7).isEqualTo(0); + assertThat(index8).isEqualTo(0); + } + } +} diff --git a/core/src/test/java/com/bloxbean/rocks/types/collection/RocksMultiBitmapTest.java b/core/src/test/java/com/bloxbean/rocks/types/collection/RocksMultiBitmapTest.java new file mode 100644 index 0000000..7a03717 --- /dev/null +++ b/core/src/test/java/com/bloxbean/rocks/types/collection/RocksMultiBitmapTest.java @@ -0,0 +1,311 @@ +package com.bloxbean.rocks.types.collection; + +import org.junit.jupiter.api.Test; +import org.roaringbitmap.RoaringBitmap; +import org.rocksdb.WriteBatch; +import org.rocksdb.WriteOptions; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +class RocksMultiBitmapTest extends RocksBaseTest { + + @Test + void setBit() { + for (int i=0; i < 100; i++) { + var rocksBitMap = new RocksMultiBitmap(rocksDBConfig, "test-bitmap-" + i, 10 * ( i + 1)); + rocksBitMap.setBit("ns1", 1); + rocksBitMap.setBit("ns1", 50); + rocksBitMap.setBit("ns1", 99); + + rocksBitMap.setBit("ns1", 101); + rocksBitMap.setBit("ns1", 150); + rocksBitMap.setBit("ns1", 199); + + rocksBitMap.setBit("ns1", 201); + rocksBitMap.setBit("ns1", 250); + + assertThat(rocksBitMap.getBit("ns1", 1)).isTrue(); + assertThat(rocksBitMap.getBit("ns1", 50)).isTrue(); + assertThat(rocksBitMap.getBit("ns1", 99)).isTrue(); + assertThat(rocksBitMap.getBit("ns1", 101)).isTrue(); + assertThat(rocksBitMap.getBit("ns1", 150)).isTrue(); + assertThat(rocksBitMap.getBit("ns1", 199)).isTrue(); + assertThat(rocksBitMap.getBit("ns1", 201)).isTrue(); + assertThat(rocksBitMap.getBit("ns1", 250)).isTrue(); + + assertThat(rocksBitMap.getBit("ns1", 41)).isFalse(); + assertThat(rocksBitMap.getBit("ns1", 100)).isFalse(); + assertThat(rocksBitMap.getBit("ns1", 200)).isFalse(); + assertThat(rocksBitMap.getBit("ns1", 240)).isFalse(); + assertThat(rocksBitMap.getBit("ns1", 350)).isFalse(); + } + } + + @Test + void setBit_batch() throws Exception { + var rocksBitMap = new RocksMultiBitmap(rocksDBConfig, "test-bitmap", 20); + WriteBatch writeBatch = new WriteBatch(); + rocksBitMap.setBitBatch("ns1", writeBatch, 1, 50, 99, 101, 150, 199, 201, 250); + + rocksDBConfig.getRocksDB().write(new WriteOptions(), writeBatch); + + assertThat(rocksBitMap.getBit("ns1", 1)).isTrue(); + assertThat(rocksBitMap.getBit("ns1", 50)).isTrue(); + assertThat(rocksBitMap.getBit("ns1", 99)).isTrue(); + assertThat(rocksBitMap.getBit("ns1", 101)).isTrue(); + assertThat(rocksBitMap.getBit("ns1", 150)).isTrue(); + assertThat(rocksBitMap.getBit("ns1", 199)).isTrue(); + assertThat(rocksBitMap.getBit("ns1", 201)).isTrue(); + assertThat(rocksBitMap.getBit("ns1", 250)).isTrue(); + + assertThat(rocksBitMap.getBit("ns1", 41)).isFalse(); + assertThat(rocksBitMap.getBit("ns1", 100)).isFalse(); + assertThat(rocksBitMap.getBit("ns1", 200)).isFalse(); + assertThat(rocksBitMap.getBit("ns1", 240)).isFalse(); + assertThat(rocksBitMap.getBit("ns1", 350)).isFalse(); + } + + @Test + void clearBit() { + var rocksBitMap = new RocksMultiBitmap(rocksDBConfig, "test-bitmap"); + rocksBitMap.setBit("ns1", 1); + rocksBitMap.setBit("ns1", 50); + rocksBitMap.setBit("ns1", 99); + + rocksBitMap.setBit("ns1", 101); + rocksBitMap.setBit("ns1", 150); + rocksBitMap.setBit("ns1", 199); + + rocksBitMap.setBit("ns1", 201); + rocksBitMap.setBit("ns1", 250); + + rocksBitMap.clearBit("ns1", 99); + rocksBitMap.clearBit("ns1", 250); + + assertThat(rocksBitMap.getBit("ns1", 1)).isTrue(); + assertThat(rocksBitMap.getBit("ns1", 50)).isTrue(); + assertThat(rocksBitMap.getBit("ns1", 99)).isFalse(); + assertThat(rocksBitMap.getBit("ns1", 101)).isTrue(); + assertThat(rocksBitMap.getBit("ns1", 150)).isTrue(); + assertThat(rocksBitMap.getBit("ns1", 199)).isTrue(); + assertThat(rocksBitMap.getBit("ns1", 201)).isTrue(); + assertThat(rocksBitMap.getBit("ns1", 250)).isFalse(); + + } + + @Test + void clearBit_batch() throws Exception { + var rocksBitMap = new RocksMultiBitmap(rocksDBConfig, "test-bitmap", 30); + rocksBitMap.setBit("ns1", 1); + rocksBitMap.setBit("ns1", 50); + rocksBitMap.setBit("ns1", 99); + + rocksBitMap.setBit("ns1", 101); + rocksBitMap.setBit("ns1", 150); + rocksBitMap.setBit("ns1", 199); + + rocksBitMap.setBit("ns1", 201); + rocksBitMap.setBit("ns1", 250); + + var writeBatch = new WriteBatch(); + rocksBitMap.clearBitBatch("ns1", writeBatch, 99, 250); + + rocksDBConfig.getRocksDB().write(new WriteOptions(), writeBatch); + + assertThat(rocksBitMap.getBit("ns1", 1)).isTrue(); + assertThat(rocksBitMap.getBit("ns1", 50)).isTrue(); + assertThat(rocksBitMap.getBit("ns1", 99)).isFalse(); + assertThat(rocksBitMap.getBit("ns1", 101)).isTrue(); + assertThat(rocksBitMap.getBit("ns1", 150)).isTrue(); + assertThat(rocksBitMap.getBit("ns1", 199)).isTrue(); + assertThat(rocksBitMap.getBit("ns1", 201)).isTrue(); + assertThat(rocksBitMap.getBit("ns1", 250)).isFalse(); + + } + + @Test + void getAllBits() { + var rocksBitMap = new RocksMultiBitmap(rocksDBConfig, "test-bitmap", 30); + rocksBitMap.setBit("ns1", 1); + rocksBitMap.setBit("ns1", 50); + rocksBitMap.setBit("ns1", 99); + + rocksBitMap.setBit("ns1", 101); + rocksBitMap.setBit("ns1", 150); + rocksBitMap.setBit("ns1", 199); + + rocksBitMap.setBit("ns1", 201); + rocksBitMap.setBit("ns1", 250); + + var allBits = rocksBitMap.getAllBits("ns1"); + + var expectedBitSet = new RoaringBitmap(); + expectedBitSet.add(1); + expectedBitSet.add(50); + expectedBitSet.add(99); + expectedBitSet.add(101); + expectedBitSet.add(150); + expectedBitSet.add(199); + expectedBitSet.add(201); + expectedBitSet.add(250); + + System.out.println(allBits); + + assertThat(allBits).isEqualTo(expectedBitSet); + } + + @Test + void getBits() { + var rocksBitMap = new RocksMultiBitmap(rocksDBConfig, "test-bitmap", 30); + rocksBitMap.setBit("ns1", 1); + rocksBitMap.setBit("ns1", 50); + rocksBitMap.setBit("ns1", 99); + + rocksBitMap.setBit("ns1", 101); + rocksBitMap.setBit("ns1", 150); + rocksBitMap.setBit("ns1", 199); + + rocksBitMap.setBit("ns1", 201); + rocksBitMap.setBit("ns1", 250); + + var bits = rocksBitMap.getBits("ns1", 2, 5); + + var expectedBits = new RoaringBitmap(); + expectedBits.add(99); + expectedBits.add(101); + expectedBits.add(150); + + assertThat(bits).isEqualTo(expectedBits); + } + + @Test + void nextSetBit() { + for (int i=0; i < 10; i++) { + var rocksBitMap = new RocksMultiBitmap(rocksDBConfig, "test-bitmap-" + i, (10 * (i + 1))); + rocksBitMap.setBit("ns1", 1); + rocksBitMap.setBit("ns1", 50); + rocksBitMap.setBit("ns1", 99); + + rocksBitMap.setBit("ns1", 101); + rocksBitMap.setBit("ns1", 150); + rocksBitMap.setBit("ns1", 199); + + rocksBitMap.setBit("ns1", 201); + rocksBitMap.setBit("ns1", 250); + + long index1 = rocksBitMap.nextSetBit("ns1", 140); + long index2 = rocksBitMap.nextSetBit("ns1", 199); + long index3 = rocksBitMap.nextSetBit("ns1", 202); + long index4 = rocksBitMap.nextSetBit("ns1", 250); + long index5 = rocksBitMap.nextSetBit("ns1", 251); + + assertThat(index1).isEqualTo(150); + assertThat(index2).isEqualTo(199); + assertThat(index3).isEqualTo(250); + assertThat(index4).isEqualTo(250); + assertThat(index5).isEqualTo(-1); + } + } + + @Test + void nextClearBit() { + for (int i=0; i < 10; i++) { + System.out.println(i); + var rocksBitMap = new RocksMultiBitmap(rocksDBConfig, "test-bitmap-" + i, 10 * (i + 1)); + rocksBitMap.setBit("ns1", 1); + rocksBitMap.setBit("ns1", 50); + rocksBitMap.setBit("ns1", 99); + + rocksBitMap.setBit("ns1", 101); + rocksBitMap.setBit("ns1", 150); + rocksBitMap.setBit("ns1", 199); + + rocksBitMap.setBit("ns1", 201); + rocksBitMap.setBit("ns1", 250); + + long index1 = rocksBitMap.nextClearBit("ns1", 140); + long index2 = rocksBitMap.nextClearBit("ns1", 199); + long index3 = rocksBitMap.nextClearBit("ns1", 202); + long index4 = rocksBitMap.nextClearBit("ns1", 250); + long index5 = rocksBitMap.nextClearBit("ns1", 251); + + assertThat(index1).isEqualTo(140); + assertThat(index2).isEqualTo(200); + assertThat(index3).isEqualTo(202); + assertThat(index4).isEqualTo(251); + assertThat(index5).isEqualTo(251); + } + } + + @Test + void prevSetBit() { + for (int i=0; i < 10; i++) { + var rocksBitMap = new RocksMultiBitmap(rocksDBConfig, "test-bitmap-" + i, 10 * (i + 1)); + rocksBitMap.setBit("ns1", 1); + rocksBitMap.setBit("ns1", 50); + rocksBitMap.setBit("ns1", 99); + + rocksBitMap.setBit("ns1", 101); + rocksBitMap.setBit("ns1", 150); + rocksBitMap.setBit("ns1", 199); + + rocksBitMap.setBit("ns1", 201); + rocksBitMap.setBit("ns1", 250); + + long index1 = rocksBitMap.previousSetBit("ns1", 140); + long index2 = rocksBitMap.previousSetBit("ns1", 199); + long index3 = rocksBitMap.previousSetBit("ns1", 202); + long index4 = rocksBitMap.previousSetBit("ns1", 250); + long index5 = rocksBitMap.previousSetBit("ns1", 251); + long index6 = rocksBitMap.previousSetBit("ns1", 49); + long index7 = rocksBitMap.previousSetBit("ns1", 1); + long index8 = rocksBitMap.previousSetBit("ns1", 0); + + assertThat(index1).isEqualTo(101); + assertThat(index2).isEqualTo(199); + assertThat(index3).isEqualTo(201); + assertThat(index4).isEqualTo(250); + assertThat(index5).isEqualTo(250); + assertThat(index6).isEqualTo(1); + assertThat(index7).isEqualTo(1); + assertThat(index8).isEqualTo(-1); + } + } + + @Test + void prevClearBit() { + for (int i=0; i < 10; i++) { + var rocksBitMap = new RocksMultiBitmap(rocksDBConfig, "test-bitmap-" + i, 10 * (i + 1)); + rocksBitMap.setBit("ns1", 1); + rocksBitMap.setBit("ns1", 50); + rocksBitMap.setBit("ns1", 99); + + rocksBitMap.setBit("ns1", 101); + rocksBitMap.setBit("ns1", 150); + rocksBitMap.setBit("ns1", 199); + + rocksBitMap.setBit("ns1", 201); + rocksBitMap.setBit("ns1", 250); + + System.out.println(i); + + long index1 = rocksBitMap.previousClearBit("ns1", 140); + long index2 = rocksBitMap.previousClearBit("ns1", 199); + long index3 = rocksBitMap.previousClearBit("ns1", 202); + long index4 = rocksBitMap.previousClearBit("ns1", 250); + long index5 = rocksBitMap.previousClearBit("ns1", 251); + long index6 = rocksBitMap.previousClearBit("ns1", 49); + long index7 = rocksBitMap.previousClearBit("ns1", 1); + long index8 = rocksBitMap.previousClearBit("ns1", 0); + + assertThat(index1).isEqualTo(140); + assertThat(index2).isEqualTo(198); + assertThat(index3).isEqualTo(202); + assertThat(index4).isEqualTo(249); + assertThat(index5).isEqualTo(251); + assertThat(index6).isEqualTo(49); + assertThat(index7).isEqualTo(0); + assertThat(index8).isEqualTo(0); + } + } +} diff --git a/gradle.properties b/gradle.properties index b471be3..c9fbf6c 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ group = com.bloxbean artifactId = rocks-types -version = 0.0.1-preview2 +version = 0.0.1-preview3 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index a924f48..6d79a15 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,6 +1,7 @@ [libraries] rocksdb="org.rocksdb:rocksdbjni:8.8.1" messagepack-jackson="org.msgpack:jackson-dataformat-msgpack:0.9.6" +roaringbitmap="org.roaringbitmap:RoaringBitmap:1.0.1" lombok = "org.projectlombok:lombok:1.18.30" slf4j-api = "org.slf4j:slf4j-api:1.7.36"