From 788d041d0146b6d0c58ed0aea2ebe8f324f45d25 Mon Sep 17 00:00:00 2001 From: Valentyn Sobol Date: Thu, 22 Aug 2024 15:59:56 +0300 Subject: [PATCH] Fix unknown classes and ERS API improvement (#264) * Fix unknown classes and ERS API improvement [jacodb-core] Specify identity for UnknownField, UnknownMethod, JcTypedFieldImpl [jacodb-core] Fix operations with map names in LmdbKeyValueStorage [jacodb-core] Avoid creation of named maps for R/O operations [jacodb-core] Force use of sortedSet by getMapNames() [jacodb-core] For an entity type, add ability to get names of used properties/blobs/links The sets of names of properties/blobs/links returned by the API consist of names ever have been set to an entity of specified type. So the sets can contain names of properties/blobs/links that could actually have been deleted in all entities of specified type. As a bonus, the way how named map are created was changed. As of now, any R/O operation doesn't implicitly create name map, whereas R/W operation does. TODO: fix implementation atop of LMDB and SQLite. [jacodb-core] Define equals()/hashcode() in JcUnknownClass & JcUnknownType [jacodb-core] Apply JcUnknownClassLookup only for instances of JcUnknownClass As of now, there is no longer an ability to get and unknown field or an unknown method for instances other than JcUnknownClass. [jacodb-api] ERS API: inherit EntityIterable from Kotlin Sequence As of now, EntityIterable is no longer Iterable, but Kotlin Sequence. All methods and properties, except iterator(), have default implementations. Binary operations are implemented as lazy sequences. * [jacodb-core] Add to JcCacheSettings ability to specify cache SPI provider --- .../api/jvm/storage/ers/EntityIterable.kt | 28 ++-- .../api/jvm/storage/ers/EntityIterableEx.kt | 107 ++++++++++++++ .../jacodb/api/jvm/storage/ers/Transaction.kt | 15 ++ .../storage/kv/PluggableKeyValueStorage.kt | 2 +- .../jacodb/api/jvm/storage/kv/Transaction.kt | 23 ++- .../jacodb/approximation/Approximations.kt | 4 +- .../jacodb/impl/JCDBSymbolsInternerImpl.kt | 2 +- .../kotlin/org/jacodb/impl/JcClasspathImpl.kt | 36 +++-- .../main/kotlin/org/jacodb/impl/JcSettings.kt | 4 +- .../impl/features/HierarchyExtension.kt | 9 +- .../kotlin/org/jacodb/impl/features/Usages.kt | 2 +- .../features/classpaths/ClasspathCache.kt | 31 ++-- .../features/classpaths/JcUnknownClass.kt | 57 +++++++- .../impl/features/classpaths/JcUnknownType.kt | 9 ++ .../impl/storage/AbstractJcDbPersistence.kt | 2 +- .../storage/PersistentLocationsRegistry.kt | 32 +++-- .../impl/storage/ers/ErsPersistenceImpl.kt | 4 +- .../ers/decorators/AbstractDecorators.kt | 3 + .../jacodb/impl/storage/ers/kv/KVEntity.kt | 55 +++---- .../ers/kv/KVEntityRelationshipStorage.kt | 136 +++++++++++------- .../impl/storage/ers/kv/KVErsTransaction.kt | 126 ++++++++++------ .../ers/ram/RAMPersistentDataContainer.kt | 24 +++- .../impl/storage/ers/ram/RAMTransaction.kt | 6 + .../ers/ram/TransactionalPersistentMap.kt | 4 + .../storage/kv/lmdb/LmdbKeyValueStorage.kt | 21 ++- .../impl/storage/kv/lmdb/LmdbTransaction.kt | 23 +-- .../storage/kv/rocks/RocksKeyValueStorage.kt | 10 +- .../impl/storage/kv/rocks/RocksTransaction.kt | 16 ++- .../storage/kv/xodus/XodusEnvironmentsEx.kt | 6 +- .../storage/kv/xodus/XodusKeyValueStorage.kt | 17 ++- .../impl/storage/kv/xodus/XodusTransaction.kt | 6 +- .../org/jacodb/impl/types/JcTypedFieldImpl.kt | 16 ++- .../org/jacodb/testing/UnknownClassesTest.kt | 19 ++- .../ers/EntityRelationshipStorageTest.kt | 20 +++ 34 files changed, 622 insertions(+), 253 deletions(-) create mode 100644 jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/storage/ers/EntityIterableEx.kt diff --git a/jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/storage/ers/EntityIterable.kt b/jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/storage/ers/EntityIterable.kt index aa0c66a11..cf25de739 100644 --- a/jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/storage/ers/EntityIterable.kt +++ b/jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/storage/ers/EntityIterable.kt @@ -16,29 +16,29 @@ package org.jacodb.api.jvm.storage.ers -interface EntityIterable : Iterable { +interface EntityIterable : Sequence { - val size: Long + val size: Long get() = toEntityIdSet().size.toLong() - val isEmpty: Boolean + val isEmpty: Boolean get() = size == 0L val isNotEmpty: Boolean get() = !isEmpty - operator fun contains(e: Entity): Boolean + operator fun contains(e: Entity): Boolean = toEntityIdSet().contains(e.id) operator fun plus(other: EntityIterable): EntityIterable = when (other) { EMPTY -> this - else -> CollectionEntityIterable(union(other)) + else -> this.union(other) } operator fun times(other: EntityIterable): EntityIterable = when (other) { EMPTY -> EMPTY - else -> CollectionEntityIterable(intersect(other)) + else -> this.intersect(other) } - operator fun minus(other: EntityIterable): EntityIterable = when(other) { + operator fun minus(other: EntityIterable): EntityIterable = when (other) { EMPTY -> this - else -> CollectionEntityIterable(subtract(other)) + else -> this.subtract(other) } fun deleteAll() = forEach { entity -> entity.delete() } @@ -49,8 +49,6 @@ interface EntityIterable : Iterable { override val size = 0L - override val isEmpty = true - override fun contains(e: Entity) = false override fun iterator(): Iterator = emptyList().iterator() @@ -64,15 +62,15 @@ interface EntityIterable : Iterable { } } -class CollectionEntityIterable(private val set: Collection) : EntityIterable { +class CollectionEntityIterable(private val c: Collection) : EntityIterable { - override val size = set.size.toLong() + override val size = c.size.toLong() - override val isEmpty = set.isEmpty() + override val isEmpty = c.isEmpty() - override fun contains(e: Entity) = e in set + override fun contains(e: Entity) = e in c - override fun iterator() = set.iterator() + override fun iterator() = c.iterator() } class EntityIdCollectionEntityIterable( diff --git a/jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/storage/ers/EntityIterableEx.kt b/jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/storage/ers/EntityIterableEx.kt new file mode 100644 index 000000000..a9dd6e3dc --- /dev/null +++ b/jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/storage/ers/EntityIterableEx.kt @@ -0,0 +1,107 @@ +/* + * Copyright 2022 UnitTestBot contributors (utbot.org) + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jacodb.api.jvm.storage.ers + +/** + * Default lazy implementations of binary operations over instances of `EntityIterable` + */ +fun EntityIterable.union(other: EntityIterable): EntityIterable { + val self = this + return object : EntityIterable { + override fun iterator(): Iterator { + return object : Iterator { + + val iterated = HashSet() + var iter = self.iterator() + var switchedToOther = false + var next: Entity? = null + + override fun hasNext(): Boolean { + advance() + return next != null + } + + override fun next(): Entity { + advance() + return (next ?: error("No more entities available")).also { next = null } + } + + private fun advance() { + if (next == null) { + if (!switchedToOther) { + if (iter.hasNext()) { + next = iter.next().also { + iterated.add(it) + } + return + } + iter = other.iterator() + switchedToOther = true + } + while (iter.hasNext()) { + iter.next().let { + if (it !in iterated) { + next = it + return + } + } + } + } + } + } + } + } +} + +fun EntityIterable.intersect(other: EntityIterable): EntityIterable { + val self = this + return object : EntityIterable { + override fun iterator(): Iterator { + val otherSet = other.toEntityIdSet() + return self.filter { it.id in otherSet }.iterator() + } + + } +} + +fun EntityIterable.subtract(other: EntityIterable): EntityIterable { + val self = this + return object : EntityIterable { + override fun iterator(): Iterator { + val otherSet = other.toEntityIdSet() + return self.filter { it.id !in otherSet }.iterator() + } + + } +} + +fun EntityIterable.toEntityIdSet(): Set { + val it = iterator() + if (!it.hasNext()) { + return emptySet() + } + val element = it.next() + if (!it.hasNext()) { + return setOf(element.id) + } + val result = LinkedHashSet() + result.add(element.id) + while (it.hasNext()) { + result.add(it.next().id) + } + return result +} \ No newline at end of file diff --git a/jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/storage/ers/Transaction.kt b/jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/storage/ers/Transaction.kt index f5ffe91cb..0a1a70680 100644 --- a/jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/storage/ers/Transaction.kt +++ b/jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/storage/ers/Transaction.kt @@ -45,6 +45,21 @@ interface Transaction : Closeable { */ fun getTypeId(type: String): Int + /** + * Returns set of property names ever being set to an entity of specified `type`. + */ + fun getPropertyNames(type: String): Set = emptySet() + + /** + * Returns set of blob names ever being set to an entity of specified `type`. + */ + fun getBlobNamesNames(type: String): Set = emptySet() + + /** + * Returns set of link names ever being set to an entity of specified `type`. + */ + fun getLinkNamesNames(type: String): Set = emptySet() + fun all(type: String): EntityIterable fun find(type: String, propertyName: String, value: T): EntityIterable diff --git a/jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/storage/kv/PluggableKeyValueStorage.kt b/jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/storage/kv/PluggableKeyValueStorage.kt index 4bcf817ea..9949b2704 100644 --- a/jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/storage/kv/PluggableKeyValueStorage.kt +++ b/jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/storage/kv/PluggableKeyValueStorage.kt @@ -38,7 +38,7 @@ abstract class PluggableKeyValueStorage : Closeable { fun delete(map: String, key: ByteArray) = transactional { txn -> txn.delete(map, key) } - fun mapSize(map: String): Long = transactional { txn -> txn.getNamedMap(map).size(txn) } + fun mapSize(map: String): Long = transactional { txn -> txn.getNamedMap(map, create = false)?.size(txn) } ?: 0L fun all(map: String): List> = readonlyTransactional { txn -> buildList { diff --git a/jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/storage/kv/Transaction.kt b/jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/storage/kv/Transaction.kt index bb67afe00..67f472cfd 100644 --- a/jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/storage/kv/Transaction.kt +++ b/jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/storage/kv/Transaction.kt @@ -26,25 +26,29 @@ interface Transaction : Closeable { val isFinished: Boolean - fun getNamedMap(name: String): NamedMap + fun getNamedMap(name: String, create: Boolean = false): NamedMap? - fun get(map: String, key: ByteArray): ByteArray? = get(getNamedMap(map), key) + fun getMapNames(): Set + + fun get(map: String, key: ByteArray): ByteArray? = getNamedMap(map, create = false)?.let { get(it, key) } fun get(map: NamedMap, key: ByteArray): ByteArray? - fun put(map: String, key: ByteArray, value: ByteArray): Boolean = put(getNamedMap(map), key, value) + fun put(map: String, key: ByteArray, value: ByteArray): Boolean = put(getNamedMap(map, create = true)!!, key, value) fun put(map: NamedMap, key: ByteArray, value: ByteArray): Boolean - fun delete(map: String, key: ByteArray): Boolean = delete(getNamedMap(map), key) + fun delete(map: String, key: ByteArray): Boolean = getNamedMap(map, create = false)?.let { delete(it, key) } == true fun delete(map: NamedMap, key: ByteArray): Boolean - fun delete(map: String, key: ByteArray, value: ByteArray): Boolean = delete(getNamedMap(map), key, value) + fun delete(map: String, key: ByteArray, value: ByteArray): Boolean = + getNamedMap(map, create = false)?.let { delete(it, key, value) } == true fun delete(map: NamedMap, key: ByteArray, value: ByteArray): Boolean - fun navigateTo(map: String, key: ByteArray? = null): Cursor = navigateTo(getNamedMap(map), key) + fun navigateTo(map: String, key: ByteArray? = null): Cursor = + getNamedMap(map, create = false)?.let { navigateTo(it, key) } ?: EmptyCursor(this) fun navigateTo(map: NamedMap, key: ByteArray? = null): Cursor @@ -64,13 +68,6 @@ interface NamedMap { fun size(txn: Transaction): Long } -object EmptyNamedMap : NamedMap { - - override val name = "EmptyNamedMap" - - override fun size(txn: Transaction) = 0L -} - abstract class TransactionDecorator(val decorated: Transaction) : Transaction by decorated fun Transaction.withFinishedState(): Transaction = WithFinishedCheckingTxn(this) diff --git a/jacodb-approximations/src/main/kotlin/org/jacodb/approximation/Approximations.kt b/jacodb-approximations/src/main/kotlin/org/jacodb/approximation/Approximations.kt index 9e9b33e7b..ff0c80bf0 100644 --- a/jacodb-approximations/src/main/kotlin/org/jacodb/approximation/Approximations.kt +++ b/jacodb-approximations/src/main/kotlin/org/jacodb/approximation/Approximations.kt @@ -93,11 +93,11 @@ object Approximations : JcFeature, JcClassExtFeature, JcInstExtFeatu }, noSqlAction = { txn -> val valueId = persistence.findSymbolId("value") - txn.find("Annotation", "nameId", approxSymbol.compressed).asSequence() + txn.find("Annotation", "nameId", approxSymbol.compressed) .filter { it.getCompressedBlob("refKind") == RefKind.CLASS.ordinal } .flatMap { annotation -> annotation.getLink("ref").let { clazz -> - annotation.getLinks("values").asSequence().map { clazz to it } + annotation.getLinks("values").map { clazz to it } } }.filter { (_, annotationValue) -> valueId == annotationValue["nameId"] diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/JCDBSymbolsInternerImpl.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/JCDBSymbolsInternerImpl.kt index efb1f5434..48810a36f 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/JCDBSymbolsInternerImpl.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/JCDBSymbolsInternerImpl.kt @@ -121,7 +121,7 @@ class JCDBSymbolsInternerImpl : JCDBSymbolsInterner, Closeable { val unwrapped = txn.unwrap if (unwrapped is KVErsTransaction) { val kvTxn = unwrapped.kvTxn - val symbolsMap = kvTxn.getNamedMap(symbolsMapName) + val symbolsMap = kvTxn.getNamedMap(symbolsMapName, create = true)!! val stringBinding = BuiltInBindingProvider.getBinding(String::class.java) val longBinding = BuiltInBindingProvider.getBinding(Long::class.java) entries.forEach { (name, id) -> diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/JcClasspathImpl.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/JcClasspathImpl.kt index 7fe291f5b..8a44c91a8 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/JcClasspathImpl.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/JcClasspathImpl.kt @@ -16,10 +16,27 @@ package org.jacodb.impl -import kotlinx.coroutines.* -import org.jacodb.api.jvm.* +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.isActive +import kotlinx.coroutines.joinAll +import kotlinx.coroutines.withContext +import org.jacodb.api.jvm.ClassSource +import org.jacodb.api.jvm.JcAnnotation +import org.jacodb.api.jvm.JcArrayType +import org.jacodb.api.jvm.JcByteCodeLocation +import org.jacodb.api.jvm.JcClassOrInterface +import org.jacodb.api.jvm.JcClasspath +import org.jacodb.api.jvm.JcClasspathExtFeature import org.jacodb.api.jvm.JcClasspathExtFeature.JcResolvedClassResult import org.jacodb.api.jvm.JcClasspathExtFeature.JcResolvedTypeResult +import org.jacodb.api.jvm.JcClasspathFeature +import org.jacodb.api.jvm.JcClasspathTask +import org.jacodb.api.jvm.JcFeatureEvent +import org.jacodb.api.jvm.JcRefType +import org.jacodb.api.jvm.JcType +import org.jacodb.api.jvm.PredefinedPrimitives +import org.jacodb.api.jvm.RegisteredLocation import org.jacodb.api.jvm.ext.JAVA_OBJECT import org.jacodb.api.jvm.ext.toType import org.jacodb.impl.bytecode.JcClassOrInterfaceImpl @@ -50,15 +67,12 @@ class JcClasspathImpl( override val registeredLocations: List = locationsRegistrySnapshot.locations override val registeredLocationIds: Set = locationsRegistrySnapshot.ids private val classpathVfs = ClasspathVfs(globalClassVFS, locationsRegistrySnapshot) - private val featuresChain = run { - val strictFeatures = features.filter { it !is UnknownClasses } - val hasUnknownClasses = strictFeatures.size != features.size - JcFeaturesChain( - strictFeatures + listOfNotNull( - JcClasspathFeatureImpl(), - UnknownClasses.takeIf { hasUnknownClasses }) - ) - } + private val featuresChain = JcFeaturesChain( + if (!features.any { it is UnknownClasses }) { + features + JcClasspathFeatureImpl() + } else { + features.filter { it !is UnknownClasses } + JcClasspathFeatureImpl() + UnknownClasses + }) override suspend fun refreshed(closeOld: Boolean): JcClasspath { return db.new(this).also { diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/JcSettings.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/JcSettings.kt index 16ee91381..3ca6fac6f 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/JcSettings.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/JcSettings.kt @@ -25,6 +25,7 @@ import org.jacodb.api.jvm.JcPersistenceImplSettings import org.jacodb.api.jvm.JcPersistenceSettings import org.jacodb.api.jvm.storage.ers.EmptyErsSettings import org.jacodb.api.jvm.storage.ers.ErsSettings +import org.jacodb.impl.caches.guava.GUAVA_CACHE_PROVIDER_ID import org.jacodb.impl.storage.SQLITE_DATABASE_PERSISTENCE_SPI import org.jacodb.impl.storage.ers.ERS_DATABASE_PERSISTENCE_SPI import org.jacodb.impl.storage.ers.kv.KV_ERS_SPI @@ -187,8 +188,8 @@ data class JcCacheSegmentSettings( enum class ValueStoreType { WEAK, SOFT, STRONG } - class JcCacheSettings { + var cacheSpiId: String = GUAVA_CACHE_PROVIDER_ID var classes: JcCacheSegmentSettings = JcCacheSegmentSettings() var types: JcCacheSegmentSettings = JcCacheSegmentSettings() var rawInstLists: JcCacheSegmentSettings = JcCacheSegmentSettings() @@ -223,7 +224,6 @@ class JcCacheSettings { flowGraphs = JcCacheSegmentSettings(maxSize = maxSize, expiration = expiration, valueStoreType = valueStoreType) } - } object JcSQLitePersistenceSettings : JcPersistenceImplSettings { diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/HierarchyExtension.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/HierarchyExtension.kt index faac1c373..020fa4c05 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/HierarchyExtension.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/HierarchyExtension.kt @@ -29,6 +29,7 @@ import org.jacodb.api.jvm.JcMethod import org.jacodb.api.jvm.ext.HierarchyExtension import org.jacodb.api.jvm.ext.JAVA_OBJECT import org.jacodb.api.jvm.ext.findDeclaredMethodOrNull +import org.jacodb.api.jvm.storage.ers.CollectionEntityIterable import org.jacodb.api.jvm.storage.ers.Entity import org.jacodb.api.jvm.storage.ers.EntityIterable import org.jacodb.api.jvm.storage.ers.Transaction @@ -96,7 +97,7 @@ internal fun JcClasspath.allClassesExceptObject(context: JCDBContext, direct: Bo }, noSqlAction = { txn -> val objectNameId = db.persistence.findSymbolId(JAVA_OBJECT) - txn.all("Class").asSequence().filter { clazz -> + txn.all("Class").filter { clazz -> (!direct || clazz.getCompressed("inherits") == null) && clazz.getCompressed("locationId") in locationIds && clazz.getCompressed("nameId") != objectNameId @@ -168,14 +169,14 @@ private class HierarchyExtensionERS(cp: JcClasspath) : HierarchyExtensionBase(cp entireHierarchy(txn, nameId, mutableSetOf()) } else { directSubClasses(txn, nameId) - }.asSequence().filter { clazz -> clazz.getCompressed("locationId") in locationIds } + }.filter { clazz -> clazz.getCompressed("locationId") in locationIds } .toClassSourceSequence(db) }.mapTo(mutableListOf()) { cp.toJcClass(it) } } } } - private fun entireHierarchy(txn: Transaction, nameId: Long, result: MutableSet): Iterable { + private fun entireHierarchy(txn: Transaction, nameId: Long, result: MutableSet): EntityIterable { val subClasses = directSubClasses(txn, nameId) if (subClasses.isNotEmpty) { result += subClasses @@ -183,7 +184,7 @@ private class HierarchyExtensionERS(cp: JcClasspath) : HierarchyExtensionBase(cp entireHierarchy(txn, clazz.getCompressed("nameId")!!, result) } } - return result + return CollectionEntityIterable(result) } private fun directSubClasses(txn: Transaction, nameId: Long): EntityIterable { diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/Usages.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/Usages.kt index 0a3e993f6..7ca126561 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/Usages.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/Usages.kt @@ -254,7 +254,7 @@ object Usages : JcFeature { noSqlAction = { txn -> val classNameIds = classNames.mapTo(mutableSetOf()) { className -> className.asSymbolId(symbolInterner) } - txn.find("Callee", "calleeNameId", name.asSymbolId(symbolInterner).compressed).asSequence() + txn.find("Callee", "calleeNameId", name.asSymbolId(symbolInterner).compressed) .filter { callee -> callee.getCompressedBlob("calleeClassId") in classNameIds && callee.getCompressed("locationId")!! in locationIds && diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/ClasspathCache.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/ClasspathCache.kt index 70143a5d5..ecf3501c3 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/ClasspathCache.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/ClasspathCache.kt @@ -38,34 +38,17 @@ import org.jacodb.impl.JcCacheSettings import org.jacodb.impl.caches.PluggableCache import org.jacodb.impl.caches.PluggableCacheProvider import org.jacodb.impl.caches.PluggableCacheStats -import org.jacodb.impl.caches.guava.GUAVA_CACHE_PROVIDER_ID import org.jacodb.impl.features.classpaths.AbstractJcInstResult.JcFlowGraphResultImpl import org.jacodb.impl.features.classpaths.AbstractJcInstResult.JcInstListResultImpl import org.jacodb.impl.features.classpaths.AbstractJcInstResult.JcRawInstListResultImpl import java.text.NumberFormat -private val PLUGGABLE_CACHE_PROVIDER_ID: String - get() = System.getProperty("org.jacodb.impl.features.classpaths.cacheProviderId", GUAVA_CACHE_PROVIDER_ID) - /** * any class cache should extend this class */ -open class ClasspathCache(settings: JcCacheSettings) : JcClasspathExtFeature, JcMethodExtFeature { - - private companion object : KLogging() { +open class ClasspathCache(settings: JcCacheSettings) : JcClasspathExtFeature, JcMethodExtFeature, KLogging() { - private val cacheProvider = PluggableCacheProvider.getProvider(PLUGGABLE_CACHE_PROVIDER_ID) - - fun newSegment(settings: JcCacheSegmentSettings): PluggableCache { - with(settings) { - return cacheProvider.newCache { - maximumSize = maxSize.toInt() - expirationDuration = expiration - valueRefType = valueStoreType - } - } - } - } + private val cacheProvider = PluggableCacheProvider.getProvider(settings.cacheSpiId) private val classesCache = newSegment(settings.classes) @@ -163,6 +146,16 @@ open class ClasspathCache(settings: JcCacheSettings) : JcClasspathExtFeature, Jc } } + private fun newSegment(settings: JcCacheSegmentSettings): PluggableCache { + with(settings) { + return cacheProvider.newCache { + maximumSize = maxSize.toInt() + expirationDuration = expiration + valueRefType = valueStoreType + } + } + } + private fun Double.forPercentages(): String { return NumberFormat.getPercentInstance().format(this) } diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/JcUnknownClass.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/JcUnknownClass.kt index ce28767d0..829a7de9c 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/JcUnknownClass.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/JcUnknownClass.kt @@ -16,7 +16,17 @@ package org.jacodb.impl.features.classpaths -import org.jacodb.api.jvm.* +import org.jacodb.api.jvm.JcClassOrInterface +import org.jacodb.api.jvm.JcClassType +import org.jacodb.api.jvm.JcClasspath +import org.jacodb.api.jvm.JcClasspathExtFeature +import org.jacodb.api.jvm.JcField +import org.jacodb.api.jvm.JcLookup +import org.jacodb.api.jvm.JcLookupExtFeature +import org.jacodb.api.jvm.JcMethod +import org.jacodb.api.jvm.JcTypedField +import org.jacodb.api.jvm.JcTypedMethod +import org.jacodb.api.jvm.TypeName import org.jacodb.api.jvm.ext.jcdbName import org.jacodb.impl.features.classpaths.AbstractJcResolvedResult.JcResolvedClassResultImpl import org.jacodb.impl.features.classpaths.virtual.JcVirtualClassImpl @@ -35,6 +45,15 @@ class JcUnknownClass(override var classpath: JcClasspath, name: String) : JcVirt initialMethods = emptyList() ) { override val lookup: JcLookup = JcUnknownClassLookup(this) + + override fun equals(other: Any?): Boolean { + if (this === other) { + return true + } + return other is JcUnknownClass && other.name == name + } + + override fun hashCode(): Int = name.hashCode() } class JcUnknownMethod( @@ -53,6 +72,7 @@ class JcUnknownMethod( ) { companion object { + fun method(type: JcClassOrInterface, name: String, access: Int, description: String): JcMethod { val methodType = Type.getMethodType(description) val returnType = TypeNameImpl(methodType.returnType.className.jcdbName()) @@ -72,6 +92,15 @@ class JcUnknownMethod( init { bind(enclosingClass) } + + override fun equals(other: Any?): Boolean { + if (this === other) { + return true + } + return other is JcUnknownMethod && description == other.description + } + + override fun hashCode(): Int = description.hashCode() } class JcUnknownField(enclosingClass: JcClassOrInterface, name: String, access: Int, type: TypeName) : @@ -92,6 +121,17 @@ class JcUnknownField(enclosingClass: JcClassOrInterface, name: String, access: I init { bind(enclosingClass) } + + override fun equals(other: Any?): Boolean { + if (this === other) { + return true + } + return other is JcUnknownField && enclosingClass == other.enclosingClass && name == other.name + } + + override fun hashCode(): Int = enclosingClass.hashCode() * 31 + name.hashCode() + + } /** @@ -179,6 +219,9 @@ object UnknownClasses : JcClasspathExtFeature { object UnknownClassMethodsAndFields : JcLookupExtFeature { override fun lookup(clazz: JcClassOrInterface): JcLookup { + if (clazz !is JcUnknownClass) { + return TrivialLookup + } return JcUnknownClassLookup(clazz) } @@ -189,3 +232,15 @@ object UnknownClassMethodsAndFields : JcLookupExtFeature { val JcClasspath.isResolveAllToUnknown: Boolean get() = isInstalled(UnknownClasses) + +private object TrivialLookup : JcLookup { + + override fun field(name: String, typeName: TypeName?, fieldKind: JcLookup.FieldKind): JcField? = null + + override fun method(name: String, description: String): JcMethod? = null + + override fun staticMethod(name: String, description: String): JcMethod? = null + + override fun specialMethod(name: String, description: String): JcMethod? = null +} + diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/JcUnknownType.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/JcUnknownType.kt index 48d59c727..b8e3bec82 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/JcUnknownType.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/JcUnknownType.kt @@ -58,6 +58,15 @@ class JcUnknownType( override val access: Int get() = Opcodes.ACC_PUBLIC + + override fun equals(other: Any?): Boolean { + if (this === other) { + return true + } + return other is JcUnknownType && other.name == name + } + + override fun hashCode(): Int = name.hashCode() } open class JcUnknownClassLookup(val clazz: JcClassOrInterface) : JcLookup { diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/AbstractJcDbPersistence.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/AbstractJcDbPersistence.kt index 71fa252aa..e568a1439 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/AbstractJcDbPersistence.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/AbstractJcDbPersistence.kt @@ -70,7 +70,7 @@ abstract class AbstractJcDbPersistence( noSqlAction = { txn -> txn.all(BytecodeLocationEntity.BYTECODE_LOCATION_ENTITY_TYPE).map { PersistentByteCodeLocationData.fromErsEntity(it) - } + }.toList() } ).mapNotNull { try { diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/PersistentLocationsRegistry.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/PersistentLocationsRegistry.kt index 8f9c7492e..e3f61fa6a 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/PersistentLocationsRegistry.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/PersistentLocationsRegistry.kt @@ -82,7 +82,7 @@ class PersistentLocationsRegistry(private val jcdb: JcDatabaseImpl) : LocationsR jcdb, PersistentByteCodeLocationData.fromErsEntity(entity) ) - } + }.toList() } ) } @@ -108,7 +108,7 @@ class PersistentLocationsRegistry(private val jcdb: JcDatabaseImpl) : LocationsR jcdb, PersistentByteCodeLocationData.fromErsEntity(entity) ) - } + }.toList() } ) } @@ -142,7 +142,8 @@ class PersistentLocationsRegistry(private val jcdb: JcDatabaseImpl) : LocationsR context.execute( sqlAction = { jooq -> jooq.update(BYTECODELOCATIONS) - .set(BYTECODELOCATIONS.STATE, LocationState.PROCESSED.ordinal).where(BYTECODELOCATIONS.ID.`in`(ids)) + .set(BYTECODELOCATIONS.STATE, LocationState.PROCESSED.ordinal) + .where(BYTECODELOCATIONS.ID.`in`(ids)) .execute() }, noSqlAction = { txn -> @@ -288,15 +289,15 @@ class PersistentLocationsRegistry(private val jcdb: JcDatabaseImpl) : LocationsR noSqlAction = { txn -> txn.getEntityOrNull(BytecodeLocationEntity.BYTECODE_LOCATION_ENTITY_TYPE, toUpdate.id) ?.let { - it.addLink( - BytecodeLocationEntity.UPDATED_LINK, - txn.getEntityOrNull( - BytecodeLocationEntity.BYTECODE_LOCATION_ENTITY_TYPE, - refreshed.id - )!! - ) - it[BytecodeLocationEntity.STATE] = LocationState.OUTDATED.ordinal - } + it.addLink( + BytecodeLocationEntity.UPDATED_LINK, + txn.getEntityOrNull( + BytecodeLocationEntity.BYTECODE_LOCATION_ENTITY_TYPE, + refreshed.id + )!! + ) + it[BytecodeLocationEntity.STATE] = LocationState.OUTDATED.ordinal + } } ) } @@ -323,7 +324,7 @@ class PersistentLocationsRegistry(private val jcdb: JcDatabaseImpl) : LocationsR noSqlAction = { txn -> txn.all(BytecodeLocationEntity.BYTECODE_LOCATION_ENTITY_TYPE) .filter { it.getLinks(BytecodeLocationEntity.UPDATED_LINK).isNotEmpty } - .map { PersistentByteCodeLocationData.fromErsEntity(it) } + .map { PersistentByteCodeLocationData.fromErsEntity(it) }.toList() } ) .filterNot { data -> snapshots.any { it.ids.contains(data.id) } } @@ -383,8 +384,9 @@ class PersistentLocationsRegistry(private val jcdb: JcDatabaseImpl) : LocationsR value = path ) .firstOrNull { it.get(BytecodeLocationEntity.FILE_SYSTEM_ID) == fileSystemId } - ?.let { PersistentByteCodeLocationData.fromErsEntity(it) - } + ?.let { + PersistentByteCodeLocationData.fromErsEntity(it) + } } ) } diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/ErsPersistenceImpl.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/ErsPersistenceImpl.kt index fce6cec14..b9a891a3c 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/ErsPersistenceImpl.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/ErsPersistenceImpl.kt @@ -224,7 +224,7 @@ class ErsPersistenceImpl( return read { context -> context.txn.find("Class", "locationId", location.id.compressed).map { it.toClassSource(db, findSymbolName(it.getCompressed("nameId") ?: throw NullPointerException())) - } + }.toList() } } @@ -245,7 +245,7 @@ class ErsPersistenceImpl( private fun findClassSourcesImpl(context: JCDBContext, cp: JcClasspath, fullName: String): Sequence { val ids = cp.registeredLocationIds return context.txn.find("Class", "nameId", findSymbolId(fullName).compressed) - .asSequence().filter { it.getCompressed("locationId") in ids } + .filter { it.getCompressed("locationId") in ids } .map { it.toClassSource(cp.db, fullName) } } diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/decorators/AbstractDecorators.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/decorators/AbstractDecorators.kt index aaa4945a8..5a926cf2c 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/decorators/AbstractDecorators.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/decorators/AbstractDecorators.kt @@ -35,6 +35,9 @@ abstract class AbstractTransactionDecorator : Transaction { override fun getEntityOrNull(id: EntityId): Entity? = delegate.getEntityOrNull(id) override fun deleteEntity(id: EntityId) = delegate.deleteEntity(id) override fun getTypeId(type: String): Int = delegate.getTypeId(type) + override fun getPropertyNames(type: String): Set = delegate.getPropertyNames(type) + override fun getBlobNamesNames(type: String): Set = delegate.getBlobNamesNames(type) + override fun getLinkNamesNames(type: String): Set = delegate.getLinkNamesNames(type) override fun all(type: String): EntityIterable = delegate.all(type) override fun find(type: String, propertyName: String, value: T): EntityIterable = delegate.find(type, propertyName, value) diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/kv/KVEntity.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/kv/KVEntity.kt index e5448feaf..9b87ce31e 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/kv/KVEntity.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/kv/KVEntity.kt @@ -26,14 +26,14 @@ import org.jacodb.api.jvm.storage.kv.forEachWithKey class KVEntity(override val id: EntityId, override val txn: KVErsTransaction) : Entity() { override fun getRawProperty(name: String): ByteArray? { - val propertiesMap = txn.ers.propertiesMap(id.typeId, name, txn.kvTxn) + val propertiesMap = txn.ers.propertiesMap(id.typeId, name, txn.kvTxn, create = false) ?: return null val keyEntry = txn.ers.longBinding.getBytesCompressed(id.instanceId) return txn.kvTxn.get(propertiesMap, keyEntry) } override fun setRawProperty(name: String, value: ByteArray?) { val kvTxn = txn.kvTxn - val propertiesMap = txn.ers.propertiesMap(id.typeId, name, kvTxn) + val propertiesMap = txn.ers.propertiesMap(id.typeId, name, kvTxn, create = true)!! val keyEntry = txn.ers.longBinding.getBytesCompressed(id.instanceId) val oldValue = kvTxn.get(propertiesMap, keyEntry) if (value != null) { @@ -42,7 +42,7 @@ class KVEntity(override val id: EntityId, override val txn: KVErsTransaction) : return } kvTxn.put(propertiesMap, keyEntry, value) - val propertiesIndex = txn.ers.propertiesIndex(id.typeId, name, kvTxn) + val propertiesIndex = txn.ers.propertiesIndex(id.typeId, name, kvTxn, create = true)!! oldValue?.let { kvTxn.delete(propertiesIndex, oldValue, keyEntry) } @@ -50,23 +50,24 @@ class KVEntity(override val id: EntityId, override val txn: KVErsTransaction) : } else { kvTxn.delete(propertiesMap, keyEntry) oldValue?.let { - kvTxn.delete(txn.ers.propertiesIndex(id.typeId, name, kvTxn), oldValue, keyEntry) + kvTxn.delete(txn.ers.propertiesIndex(id.typeId, name, kvTxn, create = true)!!, oldValue, keyEntry) } } } override fun getRawBlob(name: String): ByteArray? = txn.run { val keyEntry = ers.longBinding.getBytesCompressed(id.instanceId) - kvTxn.get(ers.blobsMap(id.typeId, name, kvTxn), keyEntry) + ers.blobsMap(id.typeId, name, kvTxn, create = false)?.let { kvTxn.get(it, keyEntry) } } override fun setRawBlob(name: String, blob: ByteArray?) { txn.run { val keyEntry = ers.longBinding.getBytesCompressed(id.instanceId) + val blobsMap = ers.blobsMap(id.typeId, name, kvTxn, create = true)!! if (blob == null) { - kvTxn.delete(ers.blobsMap(id.typeId, name, kvTxn), keyEntry) + kvTxn.delete(blobsMap, keyEntry) } else { - kvTxn.put(ers.blobsMap(id.typeId, name, kvTxn), keyEntry, blob) + kvTxn.put(blobsMap, keyEntry, blob) } } } @@ -74,20 +75,22 @@ class KVEntity(override val id: EntityId, override val txn: KVErsTransaction) : override fun getLinks(name: String): EntityIterable = txn.run { val targetTypeId = getLinkTargetType(id.typeId, name) ?: return EntityIterable.EMPTY val longBinding = ers.longBinding - val deletedMap = ers.deletedEntitiesMap(targetTypeId, txn.kvTxn) val keyEntry = longBinding.getBytesCompressed(id.instanceId) - return InstanceIdCollectionEntityIterable( - this, targetTypeId, - buildList { - kvTxn.navigateTo(ers.linkTargetsMap(id.typeId, name, kvTxn), keyEntry).use { cursor -> - cursor.forEachWithKey(keyEntry) { _, instanceIdEntry -> - if (!isDeleted(deletedMap, instanceIdEntry)) { - add(longBinding.getObjectCompressed(instanceIdEntry)) + ers.linkTargetsMap(id.typeId, name, kvTxn, create = false)?.let { linkTargetsMap -> + val deletedMap = ers.deletedEntitiesMap(targetTypeId, txn.kvTxn, create = false) + InstanceIdCollectionEntityIterable( + this, targetTypeId, + buildList { + kvTxn.navigateTo(linkTargetsMap, keyEntry).use { cursor -> + cursor.forEachWithKey(keyEntry) { _, instanceIdEntry -> + if (deletedMap == null || !isDeleted(deletedMap, instanceIdEntry)) { + add(longBinding.getObjectCompressed(instanceIdEntry)) + } } } } - } - ) + ) + } ?: EntityIterable.EMPTY } override fun addLink(name: String, targetId: EntityId): Boolean { @@ -98,14 +101,14 @@ class KVEntity(override val id: EntityId, override val txn: KVErsTransaction) : } ?: txn.run { val nameEntry = ers.stringBinding.getBytes(name) kvTxn.put( - ers.linkTargetTypesMap(id.typeId, kvTxn), + ers.linkTargetTypesMap(id.typeId, kvTxn, create = true)!!, nameEntry, ers.intBinding.getBytesCompressed(targetId.typeId) ) } return txn.run { kvTxn.put( - ers.linkTargetsMap(id.typeId, name, kvTxn), + ers.linkTargetsMap(id.typeId, name, kvTxn, create = true)!!, ers.longBinding.getBytesCompressed(id.instanceId), ers.longBinding.getBytesCompressed(targetId.instanceId) ) @@ -114,12 +117,14 @@ class KVEntity(override val id: EntityId, override val txn: KVErsTransaction) : override fun deleteLink(name: String, targetId: EntityId): Boolean { return txn.run { - val longBinding = ers.longBinding - kvTxn.delete( - ers.linkTargetsMap(id.typeId, name, kvTxn), - longBinding.getBytesCompressed(id.instanceId), - longBinding.getBytesCompressed(targetId.instanceId) - ) + ers.linkTargetsMap(id.typeId, name, kvTxn, create = false)?.let { linkTargetsMap -> + val longBinding = ers.longBinding + kvTxn.delete( + linkTargetsMap, + longBinding.getBytesCompressed(id.instanceId), + longBinding.getBytesCompressed(targetId.instanceId) + ) + } ?: false } } } \ No newline at end of file diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/kv/KVEntityRelationshipStorage.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/kv/KVEntityRelationshipStorage.kt index 6a993545e..2a36e8264 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/kv/KVEntityRelationshipStorage.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/kv/KVEntityRelationshipStorage.kt @@ -20,7 +20,6 @@ import org.jacodb.api.jvm.storage.ers.Binding import org.jacodb.api.jvm.storage.ers.ERSConflictingTransactionException import org.jacodb.api.jvm.storage.ers.EntityRelationshipStorage import org.jacodb.api.jvm.storage.ers.Transaction -import org.jacodb.api.jvm.storage.kv.EmptyNamedMap import org.jacodb.api.jvm.storage.kv.NamedMap import org.jacodb.api.jvm.storage.kv.PluggableKeyValueStorage import org.jacodb.impl.storage.ers.decorators.withAllDecorators @@ -85,7 +84,8 @@ class KVEntityRelationshipStorage(private val kvStorage: PluggableKeyValueStorag kvTxn: org.jacodb.api.jvm.storage.kv.Transaction ): Int? { entityTypes[type]?.let { return it } - return kvTxn.get(entityTypesMap(kvTxn), stringBinding.getBytes(type))?.let { typeIdEntry -> + val entityTypesMap = entityTypesMap(kvTxn, create = false) ?: return null + return kvTxn.get(entityTypesMap, stringBinding.getBytes(type))?.let { typeIdEntry -> intBinding.getObjectCompressed(typeIdEntry).also { entityTypes[type] = it } } } @@ -96,38 +96,37 @@ class KVEntityRelationshipStorage(private val kvStorage: PluggableKeyValueStorag ): Int { return entityTypes.computeIfAbsent(type) { val typeEntry = stringBinding.getBytes(type) - kvTxn.get(entityTypesMap(kvTxn), typeEntry)?.let { typeIdEntry -> + val entityTypesMap = entityTypesMap(kvTxn, create = true)!! + kvTxn.get(entityTypesMap, typeEntry)?.let { typeIdEntry -> intBinding.getObjectCompressed(typeIdEntry) } ?: run { - entityTypesMap(kvTxn).size(kvTxn).toInt().also { typeId -> - kvTxn.put(entityTypesMap(kvTxn), typeEntry, intBinding.getBytesCompressed(typeId)) + entityTypesMap.size(kvTxn).toInt().also { typeId -> + kvTxn.put(entityTypesMap, typeEntry, intBinding.getBytesCompressed(typeId)) } } } } - private fun entityTypesMap(txn: org.jacodb.api.jvm.storage.kv.Transaction): NamedMap { - return entityTypesMap ?: txn.getNamedMap(entityTypesMapName).also { - if (it !== EmptyNamedMap) { - entityTypesMap = it - } + private fun entityTypesMap(txn: org.jacodb.api.jvm.storage.kv.Transaction, create: Boolean): NamedMap? { + return entityTypesMap ?: txn.getNamedMap(entityTypesMapName, create)?.also { + entityTypesMap = it } } - internal fun entityCountersMap(txn: org.jacodb.api.jvm.storage.kv.Transaction): NamedMap { - return entityCountersMap ?: txn.getNamedMap(entityCountersMapName).also { - if (it !== EmptyNamedMap) { - entityCountersMap = it - } + internal fun entityCountersMap(txn: org.jacodb.api.jvm.storage.kv.Transaction, create: Boolean): NamedMap? { + return entityCountersMap ?: txn.getNamedMap(entityCountersMapName, create)?.also { + entityCountersMap = it } } - internal fun deletedEntitiesMap(typeId: Int, txn: org.jacodb.api.jvm.storage.kv.Transaction): NamedMap { + internal fun deletedEntitiesMap( + typeId: Int, + txn: org.jacodb.api.jvm.storage.kv.Transaction, + create: Boolean + ): NamedMap? { return deletedEntitiesMaps.getOrElse(typeId) { - txn.getNamedMap(deletedEntitiesMapName(typeId)).also { - if (it !== EmptyNamedMap) { - deletedEntitiesMaps[typeId] = it - } + txn.getNamedMap(deletedEntitiesMapName(typeId), create)?.also { + deletedEntitiesMaps[typeId] = it } } } @@ -135,47 +134,68 @@ class KVEntityRelationshipStorage(private val kvStorage: PluggableKeyValueStorag internal fun propertiesMap( typeId: Int, propName: String, - txn: org.jacodb.api.jvm.storage.kv.Transaction - ): NamedMap { + txn: org.jacodb.api.jvm.storage.kv.Transaction, + create: Boolean + ): NamedMap? { return propertiesMaps.getOrElse(typeId with propName) { - txn.getNamedMap(propertiesMapName(typeId, propName)).also { - if (it !== EmptyNamedMap) { - propertiesMaps[typeId with propName] = it - } + txn.getNamedMap(propertiesMapName(typeId, propName), create)?.also { + propertiesMaps[typeId with propName] = it } } } + /** + * Returns property name and type id if specified map name is a properties map name for this property and type id. + */ + internal fun getPropNameFromMapName(mapName: String): Pair? = + mapName.getPropertyNameFromMapName("#properties") + + /** + * Returns blob name and type id if specified map name is a blobs map name for this blob and type id. + */ + internal fun getBlobNameFromMapName(mapName: String): Pair? = + mapName.getPropertyNameFromMapName("#blobs") + + /** + * Returns link name and type id if specified map name is a linkTargets map name for this link and type id. + */ + internal fun getLinkNameFromMapName(mapName: String): Pair? = + mapName.getPropertyNameFromMapName("#link_targets$withDuplicates") + internal fun propertiesIndex( typeId: Int, propName: String, - txn: org.jacodb.api.jvm.storage.kv.Transaction - ): NamedMap { - return propertiesIndices.getOrPut(typeId with propName) { - txn.getNamedMap(propertiesIndexName(typeId, propName)) + txn: org.jacodb.api.jvm.storage.kv.Transaction, + create: Boolean + ): NamedMap? { + return propertiesIndices.getOrElse(typeId with propName) { + txn.getNamedMap(propertiesIndexName(typeId, propName), create)?.also { + propertiesIndices[typeId with propName] = it + } } } internal fun blobsMap( typeId: Int, blobName: String, - txn: org.jacodb.api.jvm.storage.kv.Transaction - ): NamedMap { + txn: org.jacodb.api.jvm.storage.kv.Transaction, + create: Boolean + ): NamedMap? { return blobsMaps.getOrElse(typeId with blobName) { - txn.getNamedMap(blobsMapName(typeId, blobName)).also { - if (it !== EmptyNamedMap) { - blobsMaps[typeId with blobName] = it - } + txn.getNamedMap(blobsMapName(typeId, blobName), create)?.also { + blobsMaps[typeId with blobName] = it } } } - internal fun linkTargetTypesMap(typeId: Int, txn: org.jacodb.api.jvm.storage.kv.Transaction): NamedMap { + internal fun linkTargetTypesMap( + typeId: Int, + txn: org.jacodb.api.jvm.storage.kv.Transaction, + create: Boolean + ): NamedMap? { return linkTargetTypesMaps.getOrElse(typeId) { - txn.getNamedMap(linkTargetTypesMapName(typeId)).also { - if (it !== EmptyNamedMap) { - linkTargetTypesMaps[typeId] = it - } + txn.getNamedMap(linkTargetTypesMapName(typeId), create)?.also { + linkTargetTypesMaps[typeId] = it } } } @@ -183,10 +203,13 @@ class KVEntityRelationshipStorage(private val kvStorage: PluggableKeyValueStorag internal fun linkTargetsMap( typeId: Int, linkName: String, - txn: org.jacodb.api.jvm.storage.kv.Transaction - ): NamedMap { - return linkTargetsMaps.getOrPut(typeId with linkName) { - txn.getNamedMap(linkTargetsMapName(typeId, linkName)) + txn: org.jacodb.api.jvm.storage.kv.Transaction, + create: Boolean + ): NamedMap? { + return linkTargetsMaps.getOrElse(typeId with linkName) { + txn.getNamedMap(linkTargetsMapName(typeId, linkName), create)?.also { + linkTargetsMaps[typeId with linkName] = it + } } } } @@ -206,25 +229,36 @@ private val entityTypesMapName = "${packageNamePrefix}entities_types" */ private val entityCountersMapName = "${packageNamePrefix}entity_counters" -private fun deletedEntitiesMapName(typeId: Int) = "${packageNamePrefix}$typeId#deleted_entities" +private fun deletedEntitiesMapName(typeId: Int) = "$packageNamePrefix$typeId#deleted_entities" /** * InstanceId (Long) -> prop value (ByteArray) */ -private fun propertiesMapName(typeId: Int, name: String) = "${packageNamePrefix}$typeId#$name#properties" +private fun propertiesMapName(typeId: Int, name: String) = "$packageNamePrefix$typeId#$name#properties" /** * InstanceId (Long) -> prop value (ByteArray) */ private fun propertiesIndexName(typeId: Int, name: String) = - "${packageNamePrefix}$typeId#$name#properties_idx$withDuplicates" + "$packageNamePrefix$typeId#$name#properties_idx$withDuplicates" /** * InstanceId (Long) -> prop value (ByteArray) */ -private fun blobsMapName(typeId: Int, name: String) = "${packageNamePrefix}$typeId#$name#blobs" +private fun blobsMapName(typeId: Int, name: String) = "$packageNamePrefix$typeId#$name#blobs" -private fun linkTargetTypesMapName(typeId: Int) = "${packageNamePrefix}$typeId#link_target_types" +private fun linkTargetTypesMapName(typeId: Int) = "$packageNamePrefix$typeId#link_target_types" private fun linkTargetsMapName(typeId: Int, name: String) = - "${packageNamePrefix}$typeId#$name#link_targets$withDuplicates" \ No newline at end of file + "$packageNamePrefix$typeId#$name#link_targets$withDuplicates" + +private fun String.getPropertyNameFromMapName(suffix: String): Pair? { + if (!startsWith(packageNamePrefix) || !endsWith(suffix)) { + return null + } + val typeAndName = substring(packageNamePrefix.length, indexOf(suffix)).split('#') + if (typeAndName.size != 2) { + error("Invalid structure of map name") + } + return typeAndName[1] to typeAndName[0].toInt() +} \ No newline at end of file diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/kv/KVErsTransaction.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/kv/KVErsTransaction.kt index 704d11e27..07f793017 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/kv/KVErsTransaction.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/kv/KVErsTransaction.kt @@ -67,7 +67,7 @@ class KVErsTransaction( val typeId = id.typeId deletedEntitiesCounts[typeId] = getDeletedEntitiesCount(typeId) + 1L kvTxn.put( - ers.deletedEntitiesMap(typeId, kvTxn), + ers.deletedEntitiesMap(typeId, kvTxn, create = true)!!, ers.longBinding.getBytesCompressed(id.instanceId), byteArrayOf(0) ) @@ -75,7 +75,9 @@ class KVErsTransaction( override fun isEntityDeleted(id: EntityId): Boolean { return getDeletedEntitiesCount(id.typeId) > 0L && isDeletedCache.getOrPut(id) { - isDeleted(ers.deletedEntitiesMap(id.typeId, kvTxn), ers.longBinding, id.instanceId) + ers.deletedEntitiesMap(id.typeId, kvTxn, create = false)?.let { + isDeleted(it, ers.longBinding, id.instanceId) + } ?: false } } @@ -83,22 +85,38 @@ class KVErsTransaction( return ers.getEntityTypeId(type, kvTxn) ?: -1 } + override fun getPropertyNames(type: String): Set = getAttributeName(type) { + ers.getPropNameFromMapName(it) + } + + override fun getBlobNamesNames(type: String): Set = getAttributeName(type) { + ers.getBlobNameFromMapName(it) + } + + override fun getLinkNamesNames(type: String): Set = getAttributeName(type) { + ers.getLinkNameFromMapName(it) + } + override fun all(type: String): EntityIterable { val typeId = ers.getEntityTypeId(type, kvTxn) ?: return EntityIterable.EMPTY val entityCounter = getEntityCounter(typeId) ?: return EntityIterable.EMPTY if (getDeletedEntitiesCount(typeId) == 0L) { return InstanceIdCollectionEntityIterable(this, typeId, (0 until entityCounter).toList()) } - val deletedMap = ers.deletedEntitiesMap(typeId, kvTxn) - return InstanceIdCollectionEntityIterable(this, typeId, - buildList { - (0 until entityCounter).forEach { instanceId -> - if (!isDeleted(deletedMap, ers.longBinding, instanceId)) { - add(instanceId) + val deletedMap = ers.deletedEntitiesMap(typeId, kvTxn, create = false) + return if (deletedMap == null) { + InstanceIdCollectionEntityIterable(this, typeId, (0 until entityCounter).toList()) + } else { + InstanceIdCollectionEntityIterable(this, typeId, + buildList { + (0 until entityCounter).forEach { instanceId -> + if (!isDeleted(deletedMap, ers.longBinding, instanceId)) { + add(instanceId) + } } } - } - ) + ) + } } override fun find(type: String, propertyName: String, value: T): EntityIterable { @@ -164,11 +182,13 @@ class KVErsTransaction( internal fun getLinkTargetType(typeId: Int, linkName: String): Int? { return linkTargetTypes.getOrElse(typeId with linkName) { val nameEntry = ers.stringBinding.getBytes(linkName) - kvTxn.get(ers.linkTargetTypesMap(typeId, kvTxn), nameEntry)?.let { typeIdEntry -> - ers.intBinding.getObjectCompressed(typeIdEntry) - }?.also { - linkTargetTypes[typeId with linkName] = it - } + ers.linkTargetTypesMap(typeId, kvTxn, create = false) + ?.let { kvTxn.get(it, nameEntry) } + ?.let { typeIdEntry -> + ers.intBinding.getObjectCompressed(typeIdEntry) + }?.also { + linkTargetTypes[typeId with linkName] = it + } } } @@ -183,35 +203,45 @@ class KVErsTransaction( return EntityIterable.EMPTY } val valueEntry = probablyCompressed(value) - return if (getDeletedEntitiesCount(typeId) == 0L) { - InstanceIdCollectionEntityIterable(this, typeId, - buildList { - kvTxn.navigateTo(ers.propertiesIndex(typeId, propertyName, kvTxn), valueEntry).use { cursor -> - cursor.cursorFun(valueEntry).forEach { (_, instanceIdEntry) -> - add(ers.longBinding.getObjectCompressed(instanceIdEntry)) + val index = ers.propertiesIndex(typeId, propertyName, kvTxn, create = false) + val deletedMap = ers.deletedEntitiesMap(typeId, kvTxn, create = false) + return if (deletedMap == null) { + if (index == null) { + EntityIterable.EMPTY + } else { + InstanceIdCollectionEntityIterable(this, typeId, + buildList { + kvTxn.navigateTo(index, valueEntry).use { cursor -> + cursor.cursorFun(valueEntry).forEach { (_, instanceIdEntry) -> + add(ers.longBinding.getObjectCompressed(instanceIdEntry)) + } } } - } - ) + ) + } } else { - val deletedMap = ers.deletedEntitiesMap(typeId, kvTxn) - InstanceIdCollectionEntityIterable(this, typeId, - buildList { - kvTxn.navigateTo(ers.propertiesIndex(typeId, propertyName, kvTxn), valueEntry).use { cursor -> - cursor.cursorFun(valueEntry).forEach { (_, instanceIdEntry) -> - if (!isDeleted(deletedMap, instanceIdEntry)) { - add(ers.longBinding.getObjectCompressed(instanceIdEntry)) + if (index == null) { + EntityIterable.EMPTY + } else { + InstanceIdCollectionEntityIterable(this, typeId, + buildList { + kvTxn.navigateTo(index, valueEntry).use { cursor -> + cursor.cursorFun(valueEntry).forEach { (_, instanceIdEntry) -> + if (!isDeleted(deletedMap, instanceIdEntry)) { + add(ers.longBinding.getObjectCompressed(instanceIdEntry)) + } } } } - } - ) + ) + } } } private fun getEntityCounter(typeId: Int): Long? { return entityCounters.getOrElse(typeId) { - kvTxn.get(ers.entityCountersMap(kvTxn), ers.intBinding.getBytesCompressed(typeId)) + ers.entityCountersMap(kvTxn, create = false) + ?.let { kvTxn.get(it, ers.intBinding.getBytesCompressed(typeId)) } ?.let { entityCounterEntry -> ers.longBinding.getObjectCompressed(entityCounterEntry) }?.also { @@ -221,19 +251,22 @@ class KVErsTransaction( } private fun getDeletedEntitiesCount(typeId: Int): Long { - return deletedEntitiesCounts.getOrPut(typeId) { - ers.deletedEntitiesMap(typeId, kvTxn).size(kvTxn) + return deletedEntitiesCounts.getOrElse(typeId) { + ers.deletedEntitiesMap(typeId, kvTxn, create = false)?.size(kvTxn) + ?.also { deletedEntitiesCounts[typeId] = it } ?: 0L } } private fun flushDirty() { - val entityCountersMap = this.ers.entityCountersMap(kvTxn) - dirtyEntityCounters.forEach { (typeId, entityCounter) -> - kvTxn.put( - entityCountersMap, - ers.intBinding.getBytesCompressed(typeId), - ers.longBinding.getBytesCompressed(entityCounter) - ) + if (dirtyEntityCounters.isNotEmpty()) { + val entityCountersMap = this.ers.entityCountersMap(kvTxn, create = true)!! + dirtyEntityCounters.forEach { (typeId, entityCounter) -> + kvTxn.put( + entityCountersMap, + ers.intBinding.getBytesCompressed(typeId), + ers.longBinding.getBytesCompressed(entityCounter) + ) + } } dirtyEntityCounters.clear() // clear caches @@ -241,4 +274,13 @@ class KVErsTransaction( linkTargetTypes.clear() entityCounters.clear() } + + private fun getAttributeName(type: String, checkMapNameFunc: (String) -> Pair?): Set { + val typeId = ers.getEntityTypeId(type, kvTxn) ?: return emptySet() + return kvTxn.getMapNames().mapNotNullTo(mutableSetOf()) { + checkMapNameFunc(it)?.let { pair -> + pair.first.takeIf { pair.second == typeId } + } + } + } } \ No newline at end of file diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/ram/RAMPersistentDataContainer.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/ram/RAMPersistentDataContainer.kt index c00987e97..b78a1395a 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/ram/RAMPersistentDataContainer.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/ram/RAMPersistentDataContainer.kt @@ -105,6 +105,12 @@ internal class RAMPersistentDataContainer( } to id } + fun getPropertyNames(type: String): Set = getAttributeNames(type, properties) + + fun getBlobNames(type: String): Set = getAttributeNames(type, blobs) + + fun getLinkNames(type: String): Set = getAttributeNames(type, links) + fun all(txn: RAMTransaction, type: String): EntityIterable { val typeId = types[type] ?: return EntityIterable.EMPTY val entities = instances[typeId] ?: return EntityIterable.EMPTY @@ -140,7 +146,6 @@ internal class RAMPersistentDataContainer( fun getRawProperty(id: EntityId, propertyName: String): ByteArray? { val typeId = id.typeId - return properties[typeId withField propertyName]?.let { it.props[id.instanceId] } } @@ -165,7 +170,7 @@ internal class RAMPersistentDataContainer( propertyName: String, value: ByteArray ): Pair { - return getEntitiesWithPropertyFunction(txn, type, propertyName) { typeId -> + return getEntitiesWithPropertyFunction(type, propertyName) { typeId -> getEntitiesWithValue(txn, typeId, value) } } @@ -176,7 +181,7 @@ internal class RAMPersistentDataContainer( propertyName: String, value: ByteArray ): Pair { - return getEntitiesWithPropertyFunction(txn, type, propertyName) { typeId -> + return getEntitiesWithPropertyFunction(type, propertyName) { typeId -> getEntitiesLtValue(txn, typeId, value) } } @@ -187,7 +192,7 @@ internal class RAMPersistentDataContainer( propertyName: String, value: ByteArray ): Pair { - return getEntitiesWithPropertyFunction(txn, type, propertyName) { typeId -> + return getEntitiesWithPropertyFunction(type, propertyName) { typeId -> getEntitiesEqOrLtValue(txn, typeId, value) } } @@ -198,7 +203,7 @@ internal class RAMPersistentDataContainer( propertyName: String, value: ByteArray ): Pair { - return getEntitiesWithPropertyFunction(txn, type, propertyName) { typeId -> + return getEntitiesWithPropertyFunction(type, propertyName) { typeId -> getEntitiesGtValue(txn, typeId, value) } } @@ -209,7 +214,7 @@ internal class RAMPersistentDataContainer( propertyName: String, value: ByteArray ): Pair { - return getEntitiesWithPropertyFunction(txn, type, propertyName) { typeId -> + return getEntitiesWithPropertyFunction(type, propertyName) { typeId -> getEntitiesEqOrGtValue(txn, typeId, value) } } @@ -270,7 +275,6 @@ internal class RAMPersistentDataContainer( } private fun getEntitiesWithPropertyFunction( - txn: RAMTransaction, type: String, propertyName: String, f: Properties.(Int) -> Pair @@ -335,6 +339,12 @@ internal class RAMPersistentDataContainer( } } } + + private fun getAttributeNames(type: String, map: TransactionalPersistentMap): Set = + types[type]?.let { typeId -> + map.getClone().entries() + .mapNotNullTo(sortedSetOf()) { if (it.key.typeId == typeId) it.key.name else null } + } ?: emptySet() } internal data class AttributeKey(val typeId: Int, val name: String) : Comparable { diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/ram/RAMTransaction.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/ram/RAMTransaction.kt index bee9b078a..39791c2c1 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/ram/RAMTransaction.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/ram/RAMTransaction.kt @@ -49,6 +49,12 @@ internal class RAMTransaction(override val ers: RAMEntityRelationshipStorage) : override fun getTypeId(type: String): Int = dataContainerChecked.getTypeId(type) + override fun getPropertyNames(type: String): Set = dataContainerChecked.getPropertyNames(type) + + override fun getBlobNamesNames(type: String): Set = dataContainerChecked.getBlobNames(type) + + override fun getLinkNamesNames(type: String): Set = dataContainerChecked.getLinkNames(type) + override fun all(type: String): EntityIterable = dataContainerChecked.all(this, type) override fun find(type: String, propertyName: String, value: T): EntityIterable { diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/ram/TransactionalPersistentMap.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/ram/TransactionalPersistentMap.kt index 3adf5ce39..0bc6e4cf5 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/ram/TransactionalPersistentMap.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/ram/TransactionalPersistentMap.kt @@ -30,6 +30,10 @@ internal class TransactionalPersistentMap(private val committed: Persisten return committed[key] } + fun entries(): Iterable> { + return (mutated ?: committed).entries + } + fun put(key: K, value: V) { mutated()[key] = value } diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/kv/lmdb/LmdbKeyValueStorage.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/kv/lmdb/LmdbKeyValueStorage.kt index 4c6d7cd5b..aac48ba24 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/kv/lmdb/LmdbKeyValueStorage.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/kv/lmdb/LmdbKeyValueStorage.kt @@ -27,14 +27,20 @@ import org.lmdbjava.Env import org.lmdbjava.EnvFlags import java.io.File import java.nio.ByteBuffer +import java.util.concurrent.ConcurrentHashMap internal class LmdbKeyValueStorage(location: String, settings: JcLmdbErsSettings) : PluggableKeyValueStorage() { + private val mapNames: MutableSet = ConcurrentHashMap().keySet(true) private val env = Env.create().apply { setMaxDbs(999999) setMaxReaders(9999999) setMapSize(settings.mapSize) - }.open(File(location), EnvFlags.MDB_NOTLS) + }.open(File(location), EnvFlags.MDB_NOTLS).apply { + dbiNames.forEach { + mapNames += String(it) + } + } override fun beginTransaction(): Transaction { return LmdbTransaction(this, env.txnWrite()).withFinishedState() @@ -48,11 +54,14 @@ internal class LmdbKeyValueStorage(location: String, settings: JcLmdbErsSettings env.close() } - internal fun getMap(lmdbTxn: org.lmdbjava.Txn, map: String): Pair, Boolean>? { + internal fun getMap( + lmdbTxn: org.lmdbjava.Txn, + map: String, + create: Boolean + ): Pair, Boolean>? { val duplicates = isMapWithKeyDuplicates?.invoke(map) == true - return if (lmdbTxn.isReadOnly) { + return if (lmdbTxn.isReadOnly || !create) { try { - env.openDbi(lmdbTxn, map.toByteArray(), null, false) to duplicates } catch (_: KeyNotFoundException) { null @@ -62,7 +71,9 @@ internal class LmdbKeyValueStorage(location: String, settings: JcLmdbErsSettings env.openDbi(lmdbTxn, map.toByteArray(), null, false, DbiFlags.MDB_CREATE, DbiFlags.MDB_DUPSORT) } else { env.openDbi(lmdbTxn, map.toByteArray(), null, false, DbiFlags.MDB_CREATE) - } to duplicates + }.also { mapNames += map } to duplicates } } + + internal fun getMapNames(): Set = mapNames.toSortedSet() } \ No newline at end of file diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/kv/lmdb/LmdbTransaction.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/kv/lmdb/LmdbTransaction.kt index 05b8fa435..fdac8b21b 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/kv/lmdb/LmdbTransaction.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/kv/lmdb/LmdbTransaction.kt @@ -17,8 +17,6 @@ package org.jacodb.impl.storage.kv.lmdb import org.jacodb.api.jvm.storage.kv.Cursor -import org.jacodb.api.jvm.storage.kv.DummyCursor -import org.jacodb.api.jvm.storage.kv.EmptyNamedMap import org.jacodb.api.jvm.storage.kv.NamedMap import org.jacodb.api.jvm.storage.kv.Transaction import org.jacodb.api.jvm.storage.kv.withFirstMoveSkipped @@ -35,23 +33,19 @@ internal class LmdbTransaction( override val isFinished: Boolean get() = lmdbTxn.id < 0 - override fun getNamedMap(name: String): NamedMap { - val (db, duplicates) = storage.getMap(lmdbTxn, name) ?: return EmptyNamedMap + override fun getNamedMap(name: String, create: Boolean): NamedMap? { + val (db, duplicates) = storage.getMap(lmdbTxn, name, create) ?: return null return LmdbNamedMap(db, duplicates, name) } + override fun getMapNames(): Set = storage.getMapNames() + override fun get(map: NamedMap, key: ByteArray): ByteArray? { - if (map === EmptyNamedMap) { - return null - } map as LmdbNamedMap return map.db.get(lmdbTxn, key.asByteBuffer)?.asArray } override fun put(map: NamedMap, key: ByteArray, value: ByteArray): Boolean { - if (map === EmptyNamedMap) { - return false - } map as LmdbNamedMap map.db.openCursor(lmdbTxn).use { cursor -> val keyBuffer = key.asByteBuffer @@ -72,9 +66,6 @@ internal class LmdbTransaction( } override fun delete(map: NamedMap, key: ByteArray): Boolean { - if (map === EmptyNamedMap) { - return false - } map as LmdbNamedMap map.db.openCursor(lmdbTxn).use { cursor -> if (cursor[key.asByteBuffer, GetOp.MDB_SET]) { @@ -86,9 +77,6 @@ internal class LmdbTransaction( } override fun delete(map: NamedMap, key: ByteArray, value: ByteArray): Boolean { - if (map === EmptyNamedMap) { - return false - } map as LmdbNamedMap map.db.openCursor(lmdbTxn).use { cursor -> if (cursor[key.asByteBuffer, value.asByteBuffer, SeekOp.MDB_GET_BOTH]) { @@ -100,9 +88,6 @@ internal class LmdbTransaction( } override fun navigateTo(map: NamedMap, key: ByteArray?): Cursor { - if (map === EmptyNamedMap) { - return DummyCursor - } map as LmdbNamedMap val cursor = map.db.openCursor(lmdbTxn) val result = LmdbCursor(this, cursor) diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/kv/rocks/RocksKeyValueStorage.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/kv/rocks/RocksKeyValueStorage.kt index 840fcbe64..7723903e1 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/kv/rocks/RocksKeyValueStorage.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/kv/rocks/RocksKeyValueStorage.kt @@ -35,6 +35,7 @@ internal abstract class RocksKeyValueStorage : PluggableKeyValueStorage() { abstract fun getNamedMapOrNull(name: String): RocksNamedMap? abstract fun getOrCreateNamedMap(name: String): RocksNamedMap + abstract fun getMapNames(): Set } internal class RocksKeyValueStorageImpl(location: String) : RocksKeyValueStorage() { @@ -78,7 +79,7 @@ internal class RocksKeyValueStorageImpl(location: String) : RocksKeyValueStorage sizesColumnFamily = getOrCreateColumnFamily( "org.jacodb.impl.storage.kv.rocks.RocksKeyValueStorage.##column##family##sizes##" ) - } catch(e: Throwable) { + } catch (e: Throwable) { columnFamilies.forEach { it.close() } rocksDB.close() throw e @@ -155,6 +156,13 @@ internal class RocksKeyValueStorageImpl(location: String) : RocksKeyValueStorage } } + override fun getMapNames(): Set { + val stringBinding = BuiltInBindingProvider.getBinding(String::class.java) + return columnFamiliesMap.mapTo(sortedSetOf()) { + stringBinding.getObject(it.key.toByteArray()) + } + } + private fun getOrCreateColumnFamily(name: ByteArray): ColumnFamilyHandle { return columnFamiliesMap[name.toList()] ?: synchronized(this) { columnFamiliesMap.getOrPut(name.toList()) { diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/kv/rocks/RocksTransaction.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/kv/rocks/RocksTransaction.kt index c6eccd907..a25beb0ba 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/kv/rocks/RocksTransaction.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/kv/rocks/RocksTransaction.kt @@ -52,10 +52,18 @@ internal class RocksTransactionImpl private constructor( // implemented using `withFinishedState()` (see `create()` in `companion object`) override val isFinished: Boolean get() = false - override fun getNamedMap(name: String): NamedMap = when { - isReadonly -> storage.getNamedMapOrNull(name) ?: LazyRocksNamedMap(name, storage) - else -> storage.getOrCreateNamedMap(name) - } + override fun getNamedMap(name: String, create: Boolean): NamedMap? = + storage.getNamedMapOrNull(name) ?: run { + if (!create) { + null + } else if (isReadonly) { + LazyRocksNamedMap(name, storage) + } else { + storage.getOrCreateNamedMap(name) + } + } + + override fun getMapNames(): Set = storage.getMapNames() override fun get(map: NamedMap, key: ByteArray): ByteArray? = (map as RocksNamedMap).get(this, key) override fun put(map: NamedMap, key: ByteArray, value: ByteArray) = (map as RocksNamedMap).put(this, key, value) diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/kv/xodus/XodusEnvironmentsEx.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/kv/xodus/XodusEnvironmentsEx.kt index 9dd89bd79..56734ade1 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/kv/xodus/XodusEnvironmentsEx.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/kv/xodus/XodusEnvironmentsEx.kt @@ -28,8 +28,12 @@ internal val ByteIterable.asByteArray: ByteArray internal fun environmentConfig(configurer: EnvironmentConfig.() -> Unit) = EnvironmentConfig().apply(configurer) +fun ObjectCacheBase.getOrElse(key: K, retriever: (K) -> V): V { + return tryKey(key) ?: retriever(key) +} + fun ObjectCacheBase.getOrPut(key: K, retriever: (K) -> V): V { - return tryKey(key) ?: retriever(key).also { obj -> cacheObject(key, obj) } + return getOrElse(key, retriever).also { obj -> cacheObject(key, obj) } } fun ObjectCacheBase.getOrPutConcurrent(key: K, retriever: (K) -> V): V { diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/kv/xodus/XodusKeyValueStorage.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/kv/xodus/XodusKeyValueStorage.kt index 270b79940..dfef8c755 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/kv/xodus/XodusKeyValueStorage.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/kv/xodus/XodusKeyValueStorage.kt @@ -46,21 +46,26 @@ internal class XodusKeyValueStorage(location: String) : PluggableKeyValueStorage env.close() } - internal fun getMap(xodusTxn: jetbrains.exodus.env.Transaction, map: String): Store { - return if (xodusTxn.isReadonly) { - env.computeInTransaction { txn -> - getMap(txn, map) - } - } else { + internal fun getMap( + xodusTxn: jetbrains.exodus.env.Transaction, + map: String, + create: Boolean + ): Store? { + return if (create || env.storeExists(map, xodusTxn)) { val duplicates = isMapWithKeyDuplicates?.invoke(map) env.openStore( map, if (duplicates == true) WITH_DUPLICATES_WITH_PREFIXING else WITHOUT_DUPLICATES_WITH_PREFIXING, xodusTxn ) + } else { + null } } + internal fun getMapNames(xodusTxn: jetbrains.exodus.env.Transaction): Set = + env.getAllStoreNames(xodusTxn).toSortedSet() + private fun jetbrains.exodus.env.Transaction.withNoStoreGetCache(): jetbrains.exodus.env.Transaction { this as TransactionBase isDisableStoreGetCache = true diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/kv/xodus/XodusTransaction.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/kv/xodus/XodusTransaction.kt index cf6851a7e..ecff5002e 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/kv/xodus/XodusTransaction.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/kv/xodus/XodusTransaction.kt @@ -31,10 +31,12 @@ internal class XodusTransaction( override val isFinished: Boolean get() = xodusTxn.isFinished - override fun getNamedMap(name: String): NamedMap { - return XodusNamedMap(storage.getMap(xodusTxn, name)) + override fun getNamedMap(name: String, create: Boolean): NamedMap? { + return storage.getMap(xodusTxn, name, create)?.let { XodusNamedMap(it) } } + override fun getMapNames(): Set = storage.getMapNames(xodusTxn) + override fun get(map: NamedMap, key: ByteArray): ByteArray? { map as XodusNamedMap return map.store.get(xodusTxn, key.asByteIterable)?.asByteArray diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/types/JcTypedFieldImpl.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/types/JcTypedFieldImpl.kt index 2d0cbb0b0..f58ad029c 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/types/JcTypedFieldImpl.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/types/JcTypedFieldImpl.kt @@ -16,8 +16,13 @@ package org.jacodb.impl.types -import org.jacodb.api.jvm.* +import org.jacodb.api.jvm.JcField +import org.jacodb.api.jvm.JcRefType +import org.jacodb.api.jvm.JcSubstitutor +import org.jacodb.api.jvm.JcType +import org.jacodb.api.jvm.JcTypedField import org.jacodb.api.jvm.ext.isNullable +import org.jacodb.api.jvm.throwClassNotFound import org.jacodb.impl.bytecode.JcAnnotationImpl import org.jacodb.impl.bytecode.JcFieldImpl import org.jacodb.impl.types.signature.FieldResolutionImpl @@ -57,4 +62,13 @@ class JcTypedFieldImpl( } ?: type } + // delegate identity to JcField + override fun equals(other: Any?): Boolean { + if (this === other) { + return true + } + return other is JcTypedFieldImpl && field == other.field + } + + override fun hashCode(): Int = field.hashCode() } diff --git a/jacodb-core/src/test/kotlin/org/jacodb/testing/UnknownClassesTest.kt b/jacodb-core/src/test/kotlin/org/jacodb/testing/UnknownClassesTest.kt index 073231d02..7880ee2f2 100644 --- a/jacodb-core/src/test/kotlin/org/jacodb/testing/UnknownClassesTest.kt +++ b/jacodb-core/src/test/kotlin/org/jacodb/testing/UnknownClassesTest.kt @@ -20,16 +20,21 @@ import org.jacodb.api.jvm.JcMethod import org.jacodb.api.jvm.ext.cfg.callExpr import org.jacodb.api.jvm.ext.cfg.fieldRef import org.jacodb.api.jvm.ext.findClass +import org.jacodb.api.jvm.ext.findDeclaredMethodOrNull +import org.jacodb.api.jvm.ext.findFieldOrNull +import org.jacodb.api.jvm.ext.findMethodOrNull +import org.jacodb.api.jvm.ext.objectClass import org.jacodb.impl.features.classpaths.JcUnknownClass import org.jacodb.impl.features.classpaths.UnknownClassMethodsAndFields import org.jacodb.impl.features.classpaths.UnknownClasses import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test class UnknownClassesTest : BaseTest() { - companion object : WithGlobalDB(UnknownClasses, UnknownClassMethodsAndFields) + companion object : WithGlobalDB(UnknownClasses) @Test fun `unknown class is resolved`() { @@ -77,6 +82,18 @@ class UnknownClassesTest : BaseTest() { } } + @Test + fun `object doesn't have unknown methods and fields`() { + cp.objectClass.let { clazz -> + assertTrue(clazz !is JcUnknownClass) + assertTrue(clazz.declaredFields.isEmpty()) + val xxxField = clazz.findFieldOrNull("xxx") + assertNull(xxxField) + val xxxMethod = clazz.findMethodOrNull("xxx", "(JILjava/lang/Exception;)V") + assertNull(xxxMethod) + } + } + private fun JcMethod.assertCfg(){ val cfg = flowGraph() cfg.instructions.forEach { diff --git a/jacodb-core/src/testFixtures/kotlin/org/jacodb/testing/storage/ers/EntityRelationshipStorageTest.kt b/jacodb-core/src/testFixtures/kotlin/org/jacodb/testing/storage/ers/EntityRelationshipStorageTest.kt index 23cb91a3a..c3f41e76c 100644 --- a/jacodb-core/src/testFixtures/kotlin/org/jacodb/testing/storage/ers/EntityRelationshipStorageTest.kt +++ b/jacodb-core/src/testFixtures/kotlin/org/jacodb/testing/storage/ers/EntityRelationshipStorageTest.kt @@ -40,6 +40,7 @@ import org.jacodb.api.jvm.storage.ers.typed.newEntity import org.jacodb.api.jvm.storage.ers.typed.property import org.jacodb.impl.storage.ers.kv.KV_ERS_SPI import org.jacodb.impl.storage.ers.ram.RAM_ERS_SPI +import org.jacodb.impl.storage.ers.sql.SQL_ERS_SPI import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Assertions.assertEquals @@ -88,6 +89,15 @@ abstract class EntityRelationshipStorageTest { val noSuchProperty: String? by propertyOf(user) assertNull(noSuchProperty) + // TODO: resolve this + if (ersSpi.id != SQL_ERS_SPI) { + txn.getPropertyNames("User").toList().let { props -> + assertEquals(2, props.size) + assertEquals("login", props[0]) + assertEquals("password", props[1]) + } + } + user.deleteProperty("login") val deletedLogin: String? by propertyOf(user, "login") assertNull(deletedLogin) @@ -110,6 +120,16 @@ abstract class EntityRelationshipStorageTest { user["seed"] = 2808.compressed user["login"] = "user2" } + + // TODO: resolve this + if (ersSpi.id != SQL_ERS_SPI) { + txn.getPropertyNames("User").toList().let { props -> + assertEquals(2, props.size) + assertEquals("login", props[0]) + assertEquals("seed", props[1]) + } + } + val found = txn.find("User", "seed", 2808.compressed) assertFalse(found.isEmpty) assertEquals(2L, found.size)