From dc7c14126425e36a775039b2aa53b6909b24bdda Mon Sep 17 00:00:00 2001 From: Eugen Stan Date: Wed, 17 May 2023 01:24:04 +0300 Subject: [PATCH] More tests for map implementation --- examples/simple-deps/deps.edn | 5 +- examples/simple-deps/src/java_map.clj | 268 ++++++++++++++++++-- examples/simple-deps/test/java_map_test.clj | 24 +- 3 files changed, 270 insertions(+), 27 deletions(-) diff --git a/examples/simple-deps/deps.edn b/examples/simple-deps/deps.edn index 151245b5..136c5b01 100644 --- a/examples/simple-deps/deps.edn +++ b/examples/simple-deps/deps.edn @@ -2,7 +2,8 @@ :deps {datalevin/datalevin #_{:git/tag "0.6.21" :git/sha "41d1915eb67b1606aab4807058ff24bb41fdb8e8"} #_{:local/root "../.."} - {:mvn/version "0.8.16"} + {:mvn/version "0.8.14"} + babashka/fs {:mvn/version "0.4.18"} com.cognitect/transit-clj {:mvn/version "1.0.329"}} :aliases {:dev {:extra-paths ["test"] :extra-deps {com.google.guava/guava-testlib {:mvn/version "31.1-jre"} @@ -16,11 +17,13 @@ com.rpl/proxy-plus {:mvn/version "0.0.9"} com.amperity/dialog {:mvn/version "2.0.115"} org.clojure/tools.logging {:mvn/version "1.2.4"} + lambdaisland/kaocha {:mvn/version "1.83.1314"} ;; allow us to run junit tests. ;; https://github.com/junit-team/junit5-samples/blob/main/junit5-migration-gradle/README.md ;; org.junit.jupiter/junit-jupiter {:mvn/version "5.9.3"} ;; guave testlib uses junit3 style of tests ;; org.junit.vintage/junit-vintage-engine {:mvn/version "5.9.3"} junit/junit {:mvn/version "4.13.2"}} + :main-opts ["-m" "kaocha.runner"] :jvm-opts ["--add-opens=java.base/java.nio=ALL-UNNAMED" "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED"]}}} \ No newline at end of file diff --git a/examples/simple-deps/src/java_map.clj b/examples/simple-deps/src/java_map.clj index 55326a03..42997d4b 100644 --- a/examples/simple-deps/src/java_map.clj +++ b/examples/simple-deps/src/java_map.clj @@ -1,6 +1,10 @@ (ns java-map (:require [datalevin.core :as d] - [datalevin.interpret :as i])) + [datalevin.interpret :as i] + [clojure.tools.logging :as log] + [clojure.string :as str]) + (:import (java.util Map$Entry + Iterator))) (defn entry-set-iterator @@ -8,15 +12,39 @@ https://docs.oracle.com/javase/8/docs/api/java/util/Iterator.html" [db dbi & _opts] - (let [state (atom {})] - (reify java.util.Iterator + (let [state (atom {:current-key nil})] + (reify Iterator (hasNext [_this] - (throw (UnsupportedOperationException. "Not implemented"))) + ;; TODO: Using an atom is not very performant. + (let [k (:current-key @state) + ;; _ (log/info "hasNext" k) + ;; if v-typa and ignore-key? is true, then return true if + new-k (if (some? k) + (d/get-first db dbi [:greater-than k] :data :ignore) + ;; when current-key is nil use :all + (d/get-first db dbi [:all k] :data :ignore)) + ;; _ (log/info "hasNext new-k" new-k) + ;; get first + new-k (get new-k 0)] + (some? new-k))) (next [_this] - (throw (UnsupportedOperationException. "Not implemented"))) + (let [k (:current-key @state) + ;; _ (log/info "next" k) + ;; if v-typa and ignore-key? is true, then return true if + kv (if (some? k) + (d/get-first db dbi [:greater-than k] :data :data) + ;; when current-key is nil use :all + (d/get-first db dbi [:all k] :data :data)) + ;; get first + new-k (when kv (nth kv 0))] + ;; (log/info "next new-k" new-k) + (swap! state assoc :current-key new-k) + ;; TODO: first and second are not performant + (clojure.lang.MapEntry. (first kv) (second kv)))) (remove [_this] - (throw (UnsupportedOperationException. "Not implemented")))))) - + (let [k (:current-key @state)] + ;; (log/trace "Remove" k) + (d/transact-kv db [:del dbi k :data])))))) (defn entry-set-view [db dbi & opts] @@ -31,10 +59,24 @@ (throw (UnsupportedOperationException. "Not implemented"))) (clear [_this] (d/clear-dbi db dbi)) - (contains [_this _o] - (throw (UnsupportedOperationException. "Not implemented"))) - (containsAll [_this _c] - (throw (UnsupportedOperationException. "Not implemented"))) + (contains [_this entry] + ;; (log/info "Contains entry" entry) + (let [key (.getKey entry) + val (.getValue entry) + kv (d/get-value db dbi key :data :data) + v (second kv)] + (= val v))) + (containsAll [_this c] + ;; (log/info "containsAll" c) + (some + (fn [entry] + (when-not (instance? Map$Entry entry) + (throw (ClassCastException. "should be Map.Entry"))) + (let [k (.getKey entry) + v (.getValue entry) + kv (d/get-value db dbi k :data :data)] + (= v kv))) + c)) (isEmpty [_this] (= 0 (d/entries db dbi))) (iterator [_this] @@ -49,7 +91,111 @@ (d/entries db dbi)))) -(defn ->map +(defn key-set-iterator + "Implemnts ^java.util.Iterator for use in keySet implementation. + + https://docs.oracle.com/javase/8/docs/api/java/util/Iterator.html" + [db dbi & _opts] + (let [state (atom {:current-key nil})] + (reify Iterator + (hasNext [_this] + ;; TODO: Using an atom is not very performant. + (let [k (:current-key @state) + ;; _ (log/info "hasNext" k) + ;; if v-typa and ignore-key? is true, then return true if + kv (if (some? k) + (d/get-first db dbi [:greater-than k] :data :ignore) + ;; when current-key is nil use :all + (d/get-first db dbi [:all k] :data :ignore)) + ;; _ (log/info "hasNext new-k" kv) + ;; get first + new-k (get kv 0)] + (some? new-k))) + (next [_this] + (let [k (:current-key @state) + ;; _ (log/info "next" k) + ;; if v-typa and ignore-key? is true, then return true if + new-k (if (some? k) + (d/get-first db dbi [:greater-than k] :data :data) + ;; when current-key is nil use :all + (d/get-first db dbi [:all k] :data :data)) + ;; get first + new-k (when new-k (nth new-k 0))] + ;; (log/info "next new-k" new-k) + (swap! state assoc :current-key new-k) + new-k)) + (remove [_this] + (let [k (:current-key @state)] + ;; (log/trace "Remove" k) + (d/transact-kv db [:del dbi k :data])))))) + +(defn key-set-view + [db dbi & opts] + (reify java.util.Set + (add [_this _e] + ;; TODO: we could implement this but we wil break the Map contract + ;; https://docs.oracle.com/javase/8/docs/api/java/util/HashMap.html#entrySet-- + (throw (UnsupportedOperationException. "Not implemented"))) + (addAll [_this _c] + ;; TODO: we could implement this but we wil break the Map contract + ;; https://docs.oracle.com/javase/8/docs/api/java/util/HashMap.html#entrySet-- + (throw (UnsupportedOperationException. "Not implemented"))) + (clear [_this] + (d/clear-dbi db dbi)) + (contains [_this key] + ;; (log/info "Contains key" key) + (let [kv (d/get-value db dbi key :data :data false)] + (some? kv))) + (containsAll [_this c] + ;; (log/info "containsAll" c) + (some + (fn [k] + (let [kv (d/get-value db dbi k :data :data false)] + (some? kv))) + c)) + (isEmpty [_this] + (= 0 (d/entries db dbi))) + (iterator [_this] + (key-set-iterator db dbi opts)) + (remove [_this _o] + (throw (UnsupportedOperationException. "Not implemented"))) + (removeAll [_this _c] + (throw (UnsupportedOperationException. "Not implemented"))) + (retainAll [_this _c] + (throw (UnsupportedOperationException. "Not implemented"))) + (size [_this] + (d/entries db dbi)))) + +(defn map-values + "Implements Collection view of map values + https://docs.oracle.com/javase/8/docs/api/java/util/Map.html#values--" + [db dbi & opts] + (reify java.util.Collection + (^boolean containsAll [_this ^java.util.Collection c] + (log/info "map-values containsAll" c) + (some + (fn [e] + (let [pred (i/inter-fn [kv] + (let [v (d/read-buffer (d/v kv))] + (= v e)))] + (some? (d/get-some db dbi pred [:all])))) + c)) + (^objects toArray [_this] + (into-array (d/get-range db dbi [:all] :data :data true))) + (^objects toArray [_this ^objects objects] + ;; TODO: try to put values in provided objects array first + (into-array (.componentType (.getClass objects)) + (d/get-range db dbi [:all] :data :data true))))) + + +(defn entry-to-str + "Entry as string key=val" + [^java.util.Map$Entry entry] + (let [k (.getKey entry) + v (.getValue entry)] + (str k "=" v))) + +(defn hashMap "Provide an implementation of java.util.Map interface backed by datalevin / LMDB. Implements java.lang.AutoCloseable and can participate in with-open. @@ -85,6 +231,40 @@ (clear [_this] (d/clear-dbi db dbi)) + ;; https://docs.oracle.com/javase/8/docs/api/java/util/Map.html#computeIfAbsent-K-java.util.function.Function- + (computeIfAbsent [_this key mapping-function] + (log/info "computeIfAbsent" key mapping-function) + (when-not (some? key) + (throw (NullPointerException.))) + (when-not (some? mapping-function) + (throw (NullPointerException.))) + (let [v (.apply mapping-function key)] + (if (some? v) + ;; v has a value so we try to add it if it's not present + (let [kv (d/get-value db dbi key :data :data false)] + (when-not (some? kv) + (d/transact-kv db [[:put dbi key v :data :data]]) + v)) + ;; v was null so we return it and do nothing + nil))) + + (computeIfPresent [_this key mapping-function] + (log/info "computeIfPresent" key mapping-function) + (when-not (some? key) + (throw (NullPointerException.))) + (when-not (some? mapping-function) + (throw (NullPointerException.))) + (let [kv (d/get-value db dbi key :data :data false)] + (if (some? kv) + ;; kv has value (present) so we compute the new value + (let [old-v (nth kv 1) + v (.apply mapping-function key old-v)] + (when (some? v) + ;; new value is present so we transact it + (d/transact-kv db [[:put dbi key v :data :data]]))) + ;; v was null so we return it and do nothing + nil))) + (containsKey [_this key] (let [kv (d/get-value db dbi key :data :data false)] (some? kv))) @@ -106,10 +286,15 @@ (= 0 (d/entries db dbi))) (keySet [_this] - (throw (UnsupportedOperationException. "Not implemented"))) + (key-set-view db dbi _opts)) (put [_this key value] - (d/transact-kv db [[:put dbi key value]])) + (log/info "Put" key "val:" value) + (when (nil? key) + (throw (NullPointerException. "Key is null."))) + (let [v (d/get-value db dbi key)] + (d/transact-kv db [[:put dbi key value]]) + v)) (putAll [_this m] (when (nil? m) @@ -117,7 +302,11 @@ (let [entries (.entrySet m) parts (partition-all put-all-batch-size entries) entry->txn (fn entry->txn [e] - [:put dbi (.getKey e) (.getValue e)])] + (let [k (.getKey e) + v (.getValue e)] + (when-not (some? k) + (throw (NullPointerException. "Null key"))) + [:put dbi k v]))] (doseq [p parts] ;; convert entries from p into transactions (let [txns (into [] (map entry->txn p))] @@ -133,8 +322,15 @@ (size [_this] (d/entries db dbi)) + ;; https://docs.oracle.com/javase/8/docs/api/java/util/Map.html#values-- (values [_this] - (throw (UnsupportedOperationException. "Not implemented"))))))) + (map-values db dbi _opts)) + + ;; https://docs.oracle.com/javase/8/docs/api/java/util/AbstractMap.html#toString-- + (toString [_this] + (let [entries (.entrySet _this) + str-entries (str/join "," (map entry-to-str entries))] + (str "{" str-entries "}"))))))) (comment @@ -144,7 +340,7 @@ (instance? datalevin.lmdb.ILMDB db) - (with-open [m (->map "/tmp/datalevin/map2" "a")] + (with-open [m (hashMap "/tmp/datalevin/map2" "a")] (println (.size m)) (println (.clear m)) (println (.size m)) @@ -153,7 +349,7 @@ (println (.containsValue m "b")) (println (.size m))) - (with-open [m (->map "/tmp/datalevin/map2" "a")] + (with-open [m (hashMap "/tmp/datalevin/map2" "a")] (println (.get m 1))) (let [pred (i/inter-fn [kv] @@ -174,7 +370,7 @@ (d/transact-kv db [[:del "map-table" 36 :data]]) - (let [my-map (->map db "map-table")] + (let [my-map (hashMap db "map-table")] (println (.get my-map 42)) (.put my-map 36 "Is the secret number") (.get my-map 36) @@ -183,5 +379,39 @@ (println (.containsKey my-map 3)) (println (.containsKey my-map 42))) + + (def db (d/open-kv "/netdava/DRE/DocSearch/workspace/projects/app-tasks/tmp/demographics-2023-05")) + + (d/list-dbis db) + (d/open-dbi db "demographic-tags") + (d/open-dbi db "semmed-tags") + + (d/open-dbi db "empty-db") + + ;; (d/get-first db "demographic-tags" [:greater-than nil] :string :data) + + (d/get-first db "demographic-tags" [:all] :string :data) + + (let [a (d/get-first db "demographic-tags" + [:all] :string :ignore) + b (d/get-first db "demographic-tags" + [:greater-than nil] :string :ignore)] + (println "a is" a) + (println "b is" b)) + + (d/get-first db "empty-db" [:all] :string :data) + + (d/get-first db "empty-db" [:greater-than nil] :string :ignore true) + + (d/get-first db "demographic-tags" [:all-back] :string :data) + + (with-open [range (d/range-seq db "demographic-tags" [:all] :string :ignore)] + (doseq [item (take 10 range)] + (println item))) + (d/close-kv db) + + + + ) \ No newline at end of file diff --git a/examples/simple-deps/test/java_map_test.clj b/examples/simple-deps/test/java_map_test.clj index 3881df54..56b4a9d8 100644 --- a/examples/simple-deps/test/java_map_test.clj +++ b/examples/simple-deps/test/java_map_test.clj @@ -89,11 +89,16 @@ (.addListener (logging-test-listener))) counter (atom 0) db (d/open-kv "/tmp/collection-size-any") + _ (d/clear-dbi db "test-dbi") my-supplier (reify Supplier (get [_this] (swap! counter inc) (d/clear-dbi db "test-dbi") - (jm/->map db "test-dbi"))) - features [CollectionSize/ANY] + (jm/hashMap db "test-dbi"))) + features [CollectionSize/ANY + ;; MapFeature/GENERAL_PURPOSE + MapFeature/SUPPORTS_PUT + MapFeature/ALLOWS_NULL_VALUES + ] suite (generate-map-test-suite "collection-size-any-str" my-supplier features) _ (.run suite result) r (test-result-info result) @@ -104,10 +109,10 @@ :failure-count])] (testing "Map is a collection - collection-size-any" (is (= {:was-successful false, - :run-count 661, - :fail+error 563, - :error-count 508, - :failure-count 55} small-result)) + :run-count 745, + :fail+error 292, + :error-count 204, + :failure-count 88} small-result)) (is (= true (:was-successful small-result)))))) @@ -115,6 +120,11 @@ (run-test test-collection-size-any) + (type (make-array java.util.Map$Entry 0)) + + (.componentType + (.getClass (make-array java.util.Map$Entry 0))) + ;; reset db (d/clear (d/open-kv "/tmp/map-test")) @@ -123,7 +133,7 @@ db (d/open-kv "/tmp/map-test") my-supplier (reify Supplier (get [_this] (swap! counter inc) - (jm/->map db (str "m-" @counter)))) + (jm/hashMap db (str "m-" @counter)))) suite (generate-map-test-suite "hash-map" my-supplier default-features) _ (.run suite result)