From 235fd6e4cf8145dfb232f9618ba4954cce32fe45 Mon Sep 17 00:00:00 2001 From: henry Date: Sat, 9 Dec 2023 14:16:32 +0900 Subject: [PATCH] Implement Mongo storage (#1261) Implement Mongo storage to query by SQL Implement validator with jsonSchema for schema table Implement data coversion between Value and Bson Add example for unsupported types Regular Expression Min key Max key JavaScript JavaScript code with scope~~ deprecated: DBPointer, Symbol, Undefined --- .github/workflows/coverage.yml | 4 +- .github/workflows/rust.yml | 8 +- Cargo.lock | 718 +++++++++++++++++- Cargo.toml | 33 +- README.md | 5 + storages/mongo-storage/Cargo.toml | 32 + storages/mongo-storage/README.md | 27 + storages/mongo-storage/src/error.rs | 40 + storages/mongo-storage/src/lib.rs | 44 ++ storages/mongo-storage/src/row/data_type.rs | 112 +++ storages/mongo-storage/src/row/key.rs | 37 + storages/mongo-storage/src/row/mod.rs | 48 ++ storages/mongo-storage/src/row/value.rs | 254 +++++++ storages/mongo-storage/src/store.rs | 277 +++++++ storages/mongo-storage/src/store_mut.rs | 276 +++++++ storages/mongo-storage/src/utils.rs | 37 + storages/mongo-storage/tests/mongo_indexes.rs | 57 ++ storages/mongo-storage/tests/mongo_storage.rs | 29 + storages/mongo-storage/tests/mongo_types.rs | 100 +++ test-suite/src/function/ifnull.rs | 8 +- test-suite/src/function/rand.rs | 2 +- 21 files changed, 2118 insertions(+), 30 deletions(-) create mode 100644 storages/mongo-storage/Cargo.toml create mode 100644 storages/mongo-storage/README.md create mode 100644 storages/mongo-storage/src/error.rs create mode 100644 storages/mongo-storage/src/lib.rs create mode 100644 storages/mongo-storage/src/row/data_type.rs create mode 100644 storages/mongo-storage/src/row/key.rs create mode 100644 storages/mongo-storage/src/row/mod.rs create mode 100644 storages/mongo-storage/src/row/value.rs create mode 100644 storages/mongo-storage/src/store.rs create mode 100644 storages/mongo-storage/src/store_mut.rs create mode 100644 storages/mongo-storage/src/utils.rs create mode 100644 storages/mongo-storage/tests/mongo_indexes.rs create mode 100644 storages/mongo-storage/tests/mongo_storage.rs create mode 100644 storages/mongo-storage/tests/mongo_types.rs diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index f50092bea..b9b704b8e 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -4,11 +4,11 @@ on: push: branches: [main, release-*] paths-ignore: - - 'docs/**' + - "docs/**" pull_request: branches: [main, release-*] paths-ignore: - - 'docs/**' + - "docs/**" env: CARGO_TERM_COLOR: always diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index a533dc880..035beda55 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -4,11 +4,11 @@ on: push: branches: [main, release-*] paths-ignore: - - 'docs/**' + - "docs/**" pull_request: branches: [main, release-*] paths-ignore: - - 'docs/**' + - "docs/**" env: CARGO_TERM_COLOR: always @@ -61,6 +61,8 @@ jobs: steps: - uses: actions/checkout@v3 - uses: Swatinem/rust-cache@v2 + - name: MongoDB in GitHub Actions + uses: supercharge/mongodb-github-action@v1.10.0 - name: Redis in GitHub Actions uses: supercharge/redis-github-action@1.7.0 with: @@ -82,4 +84,4 @@ jobs: cargo run --package gluesql --example memory_storage_usage cargo run --package gluesql --example sled_multi_threaded cargo run --package gluesql --example using_config - cargo run --package gluesql --example hello_ast_builder + cargo run --package gluesql --example hello_ast_builder diff --git a/Cargo.lock b/Cargo.lock index f40e264bb..f8efb6a08 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -35,6 +35,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" dependencies = [ "cfg-if", + "getrandom", "once_cell", "version_check", "zerocopy", @@ -92,7 +93,7 @@ dependencies = [ "polling", "rustix 0.37.27", "slab", - "socket2", + "socket2 0.4.10", "waker-fn", ] @@ -159,6 +160,18 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "base64" +version = "0.21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" + [[package]] name = "bigdecimal" version = "0.4.2" @@ -269,6 +282,28 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "bson" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58da0ae1e701ea752cc46c1bb9f39d5ecefc7395c3ecd526261a566d4f16e0c2" +dependencies = [ + "ahash 0.8.6", + "base64 0.13.1", + "bitvec", + "chrono", + "hex", + "indexmap 1.9.3", + "js-sys", + "once_cell", + "rand", + "serde", + "serde_bytes", + "serde_json", + "time", + "uuid", +] + [[package]] name = "bumpalo" version = "3.14.0" @@ -441,12 +476,27 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + [[package]] name = "core-foundation-sys" version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +[[package]] +name = "cpufeatures" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.3.2" @@ -556,6 +606,80 @@ dependencies = [ "memchr", ] +[[package]] +name = "darling" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" +dependencies = [ + "darling_core", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "data-encoding" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" + +[[package]] +name = "deranged" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "derivative" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "derive_more" +version = "0.99.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version 0.4.0", + "syn 1.0.109", +] + [[package]] name = "derive_utils" version = "0.11.2" @@ -592,6 +716,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] [[package]] @@ -637,6 +762,18 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" +[[package]] +name = "enum-as-inner" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21cdad81446a7f7dc43f6a77409efeb9733d2fa65553efef6018ef257c959b73" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -704,6 +841,12 @@ dependencies = [ "toml 0.5.11", ] +[[package]] +name = "finl_unicode" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" + [[package]] name = "fnv" version = "1.0.7" @@ -928,6 +1071,7 @@ dependencies = [ "gluesql-csv-storage", "gluesql-idb-storage", "gluesql-json-storage", + "gluesql-redis-storage", "gluesql-shared-memory-storage", "gluesql-test-suite", "gluesql-web-storage", @@ -994,7 +1138,7 @@ dependencies = [ "serde", "serde_json", "sqlparser", - "strum_macros", + "strum_macros 0.25.3", "thiserror", "uuid", ] @@ -1074,6 +1218,32 @@ dependencies = [ "uuid", ] +[[package]] +name = "gluesql-mongo-storage" +version = "0.15.0" +dependencies = [ + "async-io", + "async-trait", + "bson", + "chrono", + "futures", + "gluesql-composite-storage", + "gluesql-core", + "gluesql-test-suite", + "gluesql-utils", + "hex", + "iter-enum", + "mongodb", + "rust_decimal", + "serde", + "serde_json", + "strum", + "strum_macros 0.24.3", + "thiserror", + "tokio", + "uuid", +] + [[package]] name = "gluesql-py" version = "0.15.0" @@ -1251,6 +1421,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "home" version = "0.5.5" @@ -1260,6 +1439,17 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "hostname" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" +dependencies = [ + "libc", + "match_cfg", + "winapi", +] + [[package]] name = "iana-time-zone" version = "0.1.58" @@ -1311,6 +1501,23 @@ dependencies = [ "web-sys", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "idna" version = "0.4.0" @@ -1381,6 +1588,24 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "ipconfig" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" +dependencies = [ + "socket2 0.5.5", + "widestring", + "windows-sys", + "winreg", +] + +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + [[package]] name = "iter-enum" version = "1.1.1" @@ -1443,6 +1668,12 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + [[package]] name = "linux-raw-sys" version = "0.3.8" @@ -1471,6 +1702,27 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +[[package]] +name = "lru-cache" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" +dependencies = [ + "linked-hash-map", +] + +[[package]] +name = "match_cfg" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" + +[[package]] +name = "matches" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" + [[package]] name = "md-5" version = "0.10.6" @@ -1514,6 +1766,64 @@ dependencies = [ "adler", ] +[[package]] +name = "mio" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + +[[package]] +name = "mongodb" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c926772050c3a3f87c837626bf6135c8ca688d91d31dd39a3da547fc2bc9fe" +dependencies = [ + "async-trait", + "base64 0.13.1", + "bitflags 1.3.2", + "bson", + "chrono", + "derivative", + "derive_more", + "futures-core", + "futures-executor", + "futures-io", + "futures-util", + "hex", + "hmac", + "lazy_static", + "md-5", + "pbkdf2", + "percent-encoding", + "rand", + "rustc_version_runtime", + "rustls", + "rustls-pemfile", + "serde", + "serde_bytes", + "serde_with", + "sha-1", + "sha2", + "socket2 0.4.10", + "stringprep", + "strsim", + "take_mut", + "thiserror", + "tokio", + "tokio-rustls", + "tokio-util", + "trust-dns-proto", + "trust-dns-resolver", + "typed-builder", + "uuid", + "webpki-roots", +] + [[package]] name = "nibble_vec" version = "0.1.0" @@ -1679,6 +1989,15 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "pbkdf2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" +dependencies = [ + "digest", +] + [[package]] name = "percent-encoding" version = "2.3.0" @@ -1761,6 +2080,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -1909,6 +2234,12 @@ dependencies = [ "serde", ] +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quote" version = "1.0.33" @@ -2006,7 +2337,7 @@ dependencies = [ "percent-encoding", "ryu", "sha1_smol", - "socket2", + "socket2 0.4.10", "url", ] @@ -2077,6 +2408,30 @@ dependencies = [ "bytecheck", ] +[[package]] +name = "resolv-conf" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00" +dependencies = [ + "hostname", + "quick-error", +] + +[[package]] +name = "ring" +version = "0.17.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb0205304757e5d899b9c2e448b867ffd03ae7f988002e47cd24954391394d0b" +dependencies = [ + "cc", + "getrandom", + "libc", + "spin", + "untrusted", + "windows-sys", +] + [[package]] name = "rkyv" version = "0.7.42" @@ -2137,6 +2492,34 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver 0.9.0", +] + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver 1.0.20", +] + +[[package]] +name = "rustc_version_runtime" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d31b7153270ebf48bf91c65ae5b0c00e749c4cfad505f66530ac74950249582f" +dependencies = [ + "rustc_version 0.2.3", + "semver 0.9.0", +] + [[package]] name = "rustix" version = "0.37.27" @@ -2164,6 +2547,37 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "rustls" +version = "0.21.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9" +dependencies = [ + "log", + "ring", + "rustls-webpki", + "sct", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.5", +] + +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.14" @@ -2231,12 +2645,43 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "seahash" version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + [[package]] name = "serde" version = "1.0.190" @@ -2257,6 +2702,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "serde_bytes" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab33ec92f677585af6d88c65593ae2375adde54efdbf16d597f2cbc7a6d368ff" +dependencies = [ + "serde", +] + [[package]] name = "serde_cbor" version = "0.11.2" @@ -2284,6 +2738,7 @@ version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" dependencies = [ + "indexmap 2.1.0", "itoa", "ryu", "serde", @@ -2298,12 +2753,65 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678b5a069e50bf00ecd22d0cd8ddf7c236f68581b03db652061ed5eb13a312ff" +dependencies = [ + "serde", + "serde_with_macros", +] + +[[package]] +name = "serde_with_macros" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "sha-1" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha1_smol" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + [[package]] name = "simdutf8" version = "0.1.4" @@ -2361,6 +2869,22 @@ dependencies = [ "winapi", ] +[[package]] +name = "socket2" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + [[package]] name = "sqlparser" version = "0.39.0" @@ -2378,12 +2902,42 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e08d8363704e6c71fc928674353e6b7c23dcea9d82d7012c8faf2a3a025f8d0" +[[package]] +name = "stringprep" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb41d74e231a107a1b4ee36bd1214b11285b77768d2e3824aedafa988fd36ee6" +dependencies = [ + "finl_unicode", + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "strsim" version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "strum" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" + +[[package]] +name = "strum_macros" +version = "0.24.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 1.0.109", +] + [[package]] name = "strum_macros" version = "0.25.3" @@ -2397,6 +2951,12 @@ dependencies = [ "syn 2.0.38", ] +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + [[package]] name = "syn" version = "1.0.109" @@ -2443,6 +3003,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "take_mut" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" + [[package]] name = "tap" version = "1.0.1" @@ -2512,6 +3078,35 @@ dependencies = [ "syn 2.0.38", ] +[[package]] +name = "time" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" +dependencies = [ + "deranged", + "itoa", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" +dependencies = [ + "time-core", +] + [[package]] name = "tinytemplate" version = "1.2.1" @@ -2544,9 +3139,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" dependencies = [ "backtrace", + "bytes", + "libc", + "mio", "num_cpus", "pin-project-lite", + "signal-hook-registry", + "socket2 0.5.5", "tokio-macros", + "windows-sys", ] [[package]] @@ -2560,6 +3161,30 @@ dependencies = [ "syn 2.0.38", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +dependencies = [ + "bytes", + "futures-core", + "futures-io", + "futures-sink", + "pin-project-lite", + "tokio", +] + [[package]] name = "toml" version = "0.5.11" @@ -2603,6 +3228,62 @@ dependencies = [ "winnow", ] +[[package]] +name = "trust-dns-proto" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c31f240f59877c3d4bb3b3ea0ec5a6a0cff07323580ff8c7a605cd7d08b255d" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "enum-as-inner", + "futures-channel", + "futures-io", + "futures-util", + "idna 0.2.3", + "ipnet", + "lazy_static", + "log", + "rand", + "smallvec", + "thiserror", + "tinyvec", + "tokio", + "url", +] + +[[package]] +name = "trust-dns-resolver" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4ba72c2ea84515690c9fcef4c6c660bb9df3036ed1051686de84605b74fd558" +dependencies = [ + "cfg-if", + "futures-util", + "ipconfig", + "lazy_static", + "log", + "lru-cache", + "parking_lot 0.12.1", + "resolv-conf", + "smallvec", + "thiserror", + "tokio", + "trust-dns-proto", +] + +[[package]] +name = "typed-builder" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89851716b67b937e393b3daa8423e67ddfc4bbbf1654bcf05488e95e0828db0c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "typenum" version = "1.17.0" @@ -2648,6 +3329,12 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1766d682d402817b5ac4490b3c3002d91dfa0d22812f341609f97b08757359c" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.4.0" @@ -2655,7 +3342,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" dependencies = [ "form_urlencoded", - "idna", + "idna 0.4.0", "percent-encoding", ] @@ -2672,6 +3359,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88ad59a7560b41a70d191093a945f0b87bc1deeda46fb237479708a1d6b6cdfc" dependencies = [ "getrandom", + "serde", "wasm-bindgen", ] @@ -2803,6 +3491,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" + [[package]] name = "which" version = "4.4.2" @@ -2815,6 +3509,12 @@ dependencies = [ "rustix 0.38.21", ] +[[package]] +name = "widestring" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "653f141f39ec16bba3c5abe400a0c60da7468261cc2cbf36805022876bc721a8" + [[package]] name = "winapi" version = "0.3.9" @@ -2930,6 +3630,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys", +] + [[package]] name = "wyz" version = "0.5.1" diff --git a/Cargo.toml b/Cargo.toml index 75c210428..cc081c4ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,24 +1,24 @@ [workspace] resolver = "2" members = [ - "cli", - "core", - "pkg/rust", - "pkg/javascript", - "pkg/python", - "storages/*", - "test-suite", - "utils", + "cli", + "core", + "pkg/rust", + "pkg/javascript", + "pkg/python", + "storages/*", + "test-suite", + "utils", ] default-members = [ - "cli", - "core", - "pkg/rust", - "pkg/javascript", - "pkg/python", - "storages/*", - "test-suite", - "utils", + "cli", + "core", + "pkg/rust", + "pkg/javascript", + "pkg/python", + "storages/*", + "test-suite", + "utils", ] # ref. https://github.com/rustwasm/wasm-pack/issues/1111 @@ -48,4 +48,5 @@ composite-storage = { package = "gluesql-composite-storage", path = "./storages/ web-storage = { package = "gluesql-web-storage", path = "./storages/web-storage", version = "0.15.0" } idb-storage = { package = "gluesql-idb-storage", path = "./storages/idb-storage", version = "0.15.0" } redis-storage = { package = "gluesql-redis-storage", path = "./storages/redis-storage", version = "0.15.0" } +mongo-storage = { package = "gluesql-mongo-storage", path = "./storages/mongo-storage", version = "0.15.0" } utils = { package = "gluesql-utils", path = "./utils", version = "0.15.0" } diff --git a/README.md b/README.md index c425f3711..94fd82452 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,11 @@ Sled Storage is a persistent data storage option for GlueSQL that is built on th With GlueSQL, you can use JSONL or JSON files as a database that supports SQL and AST Builder, making it a powerful option for developers who need to work with JSON data. JSON Storage is a storage system that uses two types of files: a schema file (optional) and a data file. The schema file is written in Standard SQL and stores the structure of the table, while the data file contains the actual data and supports two file formats: `*.json` and `*.jsonl`. JSON Storage supports all DML features, but is particularly specialized for SELECT and INSERT. +### Mongo Storage + +With Mongo storage, you can use mongodb as a storage for SQL queries. You can use all the features supported by GlueSQL, such as aggregations and joins, which were previously difficult to handle on an unstructured database. In particular, you can use GlueSQL's powerful schema system on mongodb, which is as strong as an RDBMS. +To run tests, refer to [here](storages/mongo-storage/README.md) + ### Web Storage WebStorage, specifically localStorage and sessionStorage, can be used as a data storage system for GlueSQL. While WebStorage is a simple key-value database that uses string keys, GlueSQL makes it more powerful by adding support for SQL queries. This allows you to use SQL to interact with WebStorage, making it a convenient option for developers who are familiar with SQL. WebStorage can be used in JavaScript (Web) environments and Rust WebAssembly environments. diff --git a/storages/mongo-storage/Cargo.toml b/storages/mongo-storage/Cargo.toml new file mode 100644 index 000000000..d3534b74d --- /dev/null +++ b/storages/mongo-storage/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "gluesql-mongo-storage" +authors = [ + "Hyoungkwan Cho ", + "Taehoon Moon ", +] +version.workspace = true +edition.workspace = true +description.workspace = true +license.workspace = true +repository.workspace = true +documentation.workspace = true + +[dependencies] +gluesql-core.workspace = true + +async-trait = "0.1" +futures = "0.3" +thiserror = "1.0" +mongodb = "2.5.0" +bson = { version = "2.6.1", features = ["chrono-0_4"] } +chrono = { version = "0.4.26", features = ["serde", "wasmbind"] } +strum_macros = "0.24" +strum = "0.24" +rust_decimal = { version = "1", features = ["serde-str"] } + +[dev-dependencies] +test-suite.workspace = true +tokio = { version = "1", features = ["rt", "macros"] } + +[features] +test-mongo = [] \ No newline at end of file diff --git a/storages/mongo-storage/README.md b/storages/mongo-storage/README.md new file mode 100644 index 000000000..f5a31b52b --- /dev/null +++ b/storages/mongo-storage/README.md @@ -0,0 +1,27 @@ +## 🚴 MongoStorage - Mongo storage support for GlueSQL + +### ⚙️ Prerequisites + +Install & start up MongoDB + +#### 1. By Docker + +##### 1-1) Install docker + +https://docs.docker.com/engine/install/ + +##### 1-2) Start up MongoDB by docker + +``` +docker run --name mongo-glue -d -p 27017:27017 mongo +``` + +#### 2. By local installation + +https://www.mongodb.com/docs/manual/installation/ + +### 🧪 Test with features + +``` +cargo test --features test-mongo +``` diff --git a/storages/mongo-storage/src/error.rs b/storages/mongo-storage/src/error.rs new file mode 100644 index 000000000..a25c8b6b9 --- /dev/null +++ b/storages/mongo-storage/src/error.rs @@ -0,0 +1,40 @@ +use {gluesql_core::error::Error, thiserror::Error}; + +pub trait ResultExt { + fn map_storage_err(self) -> Result; +} + +impl ResultExt for std::result::Result { + fn map_storage_err(self) -> Result { + self.map_err(|e| e.to_string()).map_err(Error::StorageMsg) + } +} + +pub trait OptionExt { + fn map_storage_err(self, error: E) -> Result; +} + +impl OptionExt for std::option::Option { + fn map_storage_err(self, error: E) -> Result { + self.ok_or_else(|| error.to_string()) + .map_err(Error::StorageMsg) + } +} + +#[derive(Error, Debug)] +pub enum MongoStorageError { + #[error("invalid document")] + InvalidDocument, + + #[error("unreachable")] + Unreachable, + + #[error("unsupported bson type")] + UnsupportedBsonType, + + #[error(r#"Invalid bsonType - it should be Array eg) ["string"] or ["string", "null"]"#)] + InvalidBsonType, + + #[error("Invalid glueType - it should be type of GlueSQL Value")] + InvalidGlueType, +} diff --git a/storages/mongo-storage/src/lib.rs b/storages/mongo-storage/src/lib.rs new file mode 100644 index 000000000..8510a5283 --- /dev/null +++ b/storages/mongo-storage/src/lib.rs @@ -0,0 +1,44 @@ +pub mod error; +pub mod row; +mod store; +mod store_mut; +pub mod utils; + +pub use utils::get_collection_options; + +use { + error::ResultExt, + gluesql_core::{ + error::Result, + store::{ + AlterTable, CustomFunction, CustomFunctionMut, Index, IndexMut, Metadata, Transaction, + }, + }, + mongodb::{options::ClientOptions, Client, Database}, +}; + +pub struct MongoStorage { + pub db: Database, +} + +impl MongoStorage { + pub async fn new(conn_str: &str, db_name: &str) -> Result { + let client_options = ClientOptions::parse(conn_str).await.map_storage_err()?; + let client = Client::with_options(client_options).map_storage_err()?; + let db = client.database(db_name); + + Ok(Self { db }) + } + + pub async fn drop_database(&self) -> Result<()> { + self.db.drop(None).await.map_storage_err() + } +} + +impl Metadata for MongoStorage {} +impl AlterTable for MongoStorage {} +impl CustomFunction for MongoStorage {} +impl CustomFunctionMut for MongoStorage {} +impl Index for MongoStorage {} +impl IndexMut for MongoStorage {} +impl Transaction for MongoStorage {} diff --git a/storages/mongo-storage/src/row/data_type.rs b/storages/mongo-storage/src/row/data_type.rs new file mode 100644 index 000000000..bf8662388 --- /dev/null +++ b/storages/mongo-storage/src/row/data_type.rs @@ -0,0 +1,112 @@ +use { + gluesql_core::prelude::DataType, + strum_macros::{EnumString, IntoStaticStr}, +}; + +#[derive(IntoStaticStr, EnumString)] +pub enum BsonType { + #[strum(to_string = "double")] + Double, + #[strum(to_string = "string")] + String, + #[strum(to_string = "object")] + Object, + #[strum(to_string = "array")] + Array, + #[strum(to_string = "binData")] + Binary, + #[strum(to_string = "undefined")] + Undefined, + #[strum(to_string = "objectId")] + ObjectId, + #[strum(to_string = "bool")] + Boolean, + #[strum(to_string = "date")] + Date, + #[strum(to_string = "null")] + Null, + #[strum(to_string = "regex")] + RegularExpression, + #[strum(to_string = "dbPointer")] + DbPointer, + #[strum(to_string = "javascript")] + JavaScript, + #[strum(to_string = "symbol")] + Symbol, + #[strum(to_string = "javascriptWithScope")] + JavaScriptCodeWithScope, + #[strum(to_string = "int")] + Int32, + #[strum(to_string = "timestamp")] + Timestamp, + #[strum(to_string = "long")] + Int64, + #[strum(to_string = "decimal")] + Decimal128, + #[strum(to_string = "minKey")] + MinKey, + #[strum(to_string = "maxKey")] + MaxKey, +} + +impl From<&DataType> for BsonType { + fn from(data_type: &DataType) -> BsonType { + match data_type { + DataType::Boolean => BsonType::Boolean, + DataType::Int8 => BsonType::Int32, + DataType::Int16 => BsonType::Int32, + DataType::Int32 => BsonType::Int32, + DataType::Int => BsonType::Int64, + DataType::Int128 => BsonType::Decimal128, + DataType::Uint8 => BsonType::Int32, + DataType::Uint16 => BsonType::Int32, + DataType::Uint32 => BsonType::Int64, + DataType::Uint64 => BsonType::Decimal128, + DataType::Uint128 => BsonType::Decimal128, + DataType::Float32 => BsonType::Double, + DataType::Float => BsonType::Double, + DataType::Text => BsonType::String, + DataType::Bytea => BsonType::Binary, + DataType::Date => BsonType::Date, + DataType::Timestamp => BsonType::String, + DataType::Time => BsonType::Date, + DataType::Uuid => BsonType::Binary, + DataType::Map => BsonType::Object, + DataType::List => BsonType::Array, + DataType::Decimal => BsonType::Decimal128, + DataType::Point => BsonType::Object, + DataType::Inet => BsonType::String, + DataType::Interval => BsonType::String, + } + } +} + +pub const B15: i64 = 2_i64.pow(15); +pub const B7: i64 = 2_i64.pow(7); +pub const B31: i64 = 2_i64.pow(31); +pub const TIME: i64 = 86400000 - 1; + +pub trait IntoRange { + fn get_max(&self) -> Option; + fn get_min(&self) -> Option; +} + +impl IntoRange for DataType { + fn get_max(&self) -> Option { + match self { + DataType::Int8 => Some(B7), + DataType::Int16 => Some(B15), + DataType::Int32 => Some(B31), + DataType::Float32 => Some(B31), + DataType::Time => Some(TIME), + _ => None, + } + } + + fn get_min(&self) -> Option { + match self { + DataType::Time => Some(0), + v => v.get_max().map(|max| -max), + } + } +} diff --git a/storages/mongo-storage/src/row/key.rs b/storages/mongo-storage/src/row/key.rs new file mode 100644 index 000000000..a3b339533 --- /dev/null +++ b/storages/mongo-storage/src/row/key.rs @@ -0,0 +1,37 @@ +use { + crate::error::MongoStorageError, + bson::{Binary, Bson}, + gluesql_core::prelude::{Error, Key, Result}, +}; + +pub trait KeyIntoBson { + fn into_bson(self, has_primary: bool) -> Result; +} + +impl KeyIntoBson for Key { + fn into_bson(self, has_primary: bool) -> Result { + match has_primary { + true => Ok(Bson::Binary(Binary { + subtype: bson::spec::BinarySubtype::Generic, + bytes: self.to_cmp_be_bytes()?, + })), + false => into_object_id(self), + } + } +} + +pub fn into_object_id(key: Key) -> Result { + Ok(match key { + Key::Bytea(bytes) => { + let mut byte_array: [u8; 12] = [0; 12]; + byte_array[..].copy_from_slice(&bytes[..]); + + Bson::ObjectId(bson::oid::ObjectId::from_bytes(byte_array)) + } + _ => { + return Err(Error::StorageMsg( + MongoStorageError::UnsupportedBsonType.to_string(), + )) + } + }) +} diff --git a/storages/mongo-storage/src/row/mod.rs b/storages/mongo-storage/src/row/mod.rs new file mode 100644 index 000000000..f2c63571f --- /dev/null +++ b/storages/mongo-storage/src/row/mod.rs @@ -0,0 +1,48 @@ +pub mod data_type; +pub mod key; +pub mod value; + +use { + self::value::IntoValue, + crate::error::ResultExt, + gluesql_core::{ + prelude::{DataType, Key, Result}, + store::DataRow, + }, + mongodb::bson::Document, +}; + +pub trait IntoRow { + fn into_row<'a>( + self, + data_types: impl Iterator, + is_primary: bool, + ) -> Result<(Key, DataRow)>; +} + +impl IntoRow for Document { + fn into_row<'a>( + self, + data_types: impl Iterator, + has_primary: bool, + ) -> Result<(Key, DataRow)> { + let key = match has_primary { + true => self.get_binary_generic("_id").map_storage_err()?.to_owned(), + false => self + .get_object_id("_id") + .map_storage_err()? + .bytes() + .to_vec(), + }; + let key = Key::Bytea(key); + + let row = self + .into_iter() + .skip(1) + .zip(data_types) + .map(|((_, bson), data_type)| bson.into_value(data_type)) + .collect::>>()?; + + Ok((key, DataRow::Vec(row))) + } +} diff --git a/storages/mongo-storage/src/row/value.rs b/storages/mongo-storage/src/row/value.rs new file mode 100644 index 000000000..228b9d3e1 --- /dev/null +++ b/storages/mongo-storage/src/row/value.rs @@ -0,0 +1,254 @@ +use { + crate::error::{MongoStorageError, OptionExt, ResultExt}, + gluesql_core::{ + ast::{Expr, ToSql}, + chrono::{NaiveDate, NaiveDateTime, TimeZone, Utc}, + data::{Interval, Point}, + parse_sql::parse_interval, + prelude::{DataType, Error}, + translate::translate_expr, + {data::Value, prelude::Result}, + }, + mongodb::bson::{self, doc, Binary, Bson, DateTime, Decimal128, Document}, + rust_decimal::Decimal, + std::collections::HashMap, +}; + +pub trait IntoValue { + fn into_value_schemaless(self) -> Result; + fn into_value(self, data_type: &DataType) -> Result; +} + +impl IntoValue for Bson { + fn into_value_schemaless(self) -> Result { + Ok(match self { + Bson::String(string) => Value::Str(string), + Bson::Document(d) => Value::Map( + d.into_iter() + .map(|(k, v)| Ok((k, v.into_value_schemaless()?))) + .collect::>>()?, + ), + Bson::Boolean(b) => Value::Bool(b), + Bson::Int32(i) => Value::I32(i), + Bson::Int64(i) => Value::I64(i), + _ => { + return Err(Error::StorageMsg( + MongoStorageError::UnsupportedBsonType.to_string(), + )); + } + }) + } + + fn into_value(self, data_type: &DataType) -> Result { + Ok(match (self, data_type) { + (Bson::Null, _) => Value::Null, + (Bson::Double(num), DataType::Float32) => Value::F32(num as f32), + (Bson::Double(num), _) => Value::F64(num), + (Bson::String(string), DataType::Inet) => { + Value::Inet(string.parse().map_storage_err()?) + } + (Bson::String(string), DataType::Timestamp) => Value::Timestamp( + NaiveDateTime::parse_from_str(&string, "%Y-%m-%d %H:%M:%S%.f").map_storage_err()?, + ), + (Bson::String(string), DataType::Interval) => { + let interval = parse_interval(string)?; + let interval = translate_expr(&interval)?; + match interval { + Expr::Interval { + expr, + leading_field, + last_field, + } => Value::Interval(Interval::try_from_str( + &expr.to_sql(), + leading_field, + last_field, + )?), + _ => { + return Err(Error::StorageMsg( + MongoStorageError::UnsupportedBsonType.to_string(), + )) + } + } + } + (Bson::String(string), _) => Value::Str(string), + (Bson::Array(array), _) => { + let values = array + .into_iter() + .map(|bson| bson.into_value(data_type)) + .collect::>>()?; + + Value::List(values) + } + (Bson::Document(d), DataType::Point) => { + let x = d + .get("x") + .and_then(Bson::as_f64) + .map_storage_err(MongoStorageError::UnsupportedBsonType)?; + let y = d + .get("y") + .and_then(Bson::as_f64) + .map_storage_err(MongoStorageError::UnsupportedBsonType)?; + + Value::Point(Point::new(x, y)) + } + (Bson::Document(d), _) => Value::Map( + d.into_iter() + .map(|(k, v)| Ok((k, v.into_value(data_type)?))) + .collect::>>()?, + ), + (Bson::Boolean(b), _) => Value::Bool(b), + (Bson::RegularExpression(regex), _) => { + let pattern = regex.pattern; + let options = regex.options; + Value::Str(format!("/{}/{}", pattern, options)) + } + (Bson::Int32(i), DataType::Uint8) => Value::U8(i.try_into().map_storage_err()?), + (Bson::Int32(i), DataType::Int8) => Value::I8(i as i8), + (Bson::Int32(i), DataType::Int16) => Value::I16(i as i16), + (Bson::Int32(i), DataType::Uint16) => Value::U16(i.try_into().map_storage_err()?), + (Bson::Int32(i), _) => Value::I32(i), + (Bson::Int64(i), DataType::Uint32) => Value::U32(i.try_into().map_storage_err()?), + (Bson::Int64(i), _) => Value::I64(i), + (Bson::Binary(Binary { bytes, .. }), DataType::Uuid) => { + let u128 = u128::from_be_bytes( + bytes + .try_into() + .ok() + .map_storage_err(MongoStorageError::UnsupportedBsonType)?, + ); + + Value::Uuid(u128) + } + (Bson::Binary(Binary { bytes, .. }), _) => Value::Bytea(bytes), + (Bson::Decimal128(decimal128), DataType::Uint64) => { + let bytes = decimal128.bytes(); + let u64 = u64::from_be_bytes(bytes[..8].try_into().map_storage_err()?); + + Value::U64(u64) + } + (Bson::Decimal128(decimal128), DataType::Uint128) => { + let bytes = decimal128.bytes(); + let u128 = u128::from_be_bytes(bytes); + + Value::U128(u128) + } + (Bson::Decimal128(decimal128), DataType::Int128) => { + let bytes = decimal128.bytes(); + let i128 = i128::from_be_bytes(bytes); + + Value::I128(i128) + } + (Bson::Decimal128(decimal128), _) => { + let decimal = Decimal::deserialize(decimal128.bytes()); + + Value::Decimal(decimal) + } + (Bson::DateTime(dt), DataType::Time) => Value::Time(dt.to_chrono().time()), + (Bson::DateTime(dt), _) => Value::Date(dt.to_chrono().date_naive()), + (Bson::JavaScriptCode(code), _) => Value::Str(code), + (Bson::JavaScriptCodeWithScope(bson::JavaScriptCodeWithScope { code, scope }), _) => { + Value::Map(HashMap::from([ + ("code".to_owned(), Value::Str(code)), + ( + "scope".to_owned(), + bson::to_bson(&scope) + .map_storage_err()? + .into_value_schemaless()?, + ), + ])) + } + (Bson::MinKey, _) => Value::Str("MinKey()".to_owned()), + (Bson::MaxKey, _) => Value::Str("MaxKey()".to_owned()), + _ => { + return Err(Error::StorageMsg( + MongoStorageError::UnsupportedBsonType.to_string(), + )); + } + }) + } +} + +pub trait IntoBson { + fn into_bson(self) -> Result; +} + +impl IntoBson for Value { + fn into_bson(self) -> Result { + match self { + Value::Null => Ok(Bson::Null), + Value::I32(val) => Ok(Bson::Int32(val)), + Value::I64(val) => Ok(Bson::Int64(val)), + Value::F64(val) => Ok(Bson::Double(val)), + Value::Bool(val) => Ok(Bson::Boolean(val)), + Value::Str(val) => Ok(Bson::String(val)), + Value::List(val) => { + let bson = val + .into_iter() + .map(|val| val.into_bson()) + .collect::>>()?; + + Ok(Bson::Array(bson)) + } + Value::Bytea(bytes) => Ok(Bson::Binary(bson::Binary { + subtype: bson::spec::BinarySubtype::Generic, + bytes, + })), + Value::Decimal(decimal) => Ok(Bson::Decimal128(Decimal128::from_bytes( + decimal.serialize(), + ))), + Value::I8(val) => Ok(Bson::Int32(val.into())), + Value::F32(val) => Ok(Bson::Double(val.into())), + Value::Uuid(val) => Ok(Bson::Binary(Binary { + subtype: bson::spec::BinarySubtype::Uuid, + + bytes: val.to_be_bytes().to_vec(), + })), + Value::Date(val) => { + let utc = Utc.from_utc_datetime( + &val.and_hms_opt(0, 0, 0) + .map_storage_err(MongoStorageError::UnsupportedBsonType)?, + ); + let datetime = DateTime::from_chrono(utc); + + Ok(Bson::DateTime(datetime)) + } + Value::Timestamp(val) => Ok(Bson::String(val.to_string())), + Value::Time(val) => { + let date = NaiveDate::from_ymd_opt(1970, 1, 1) + .map_storage_err(MongoStorageError::UnsupportedBsonType)?; + let utc = Utc.from_utc_datetime(&NaiveDateTime::new(date, val)); + let datetime = DateTime::from_chrono(utc); + + Ok(Bson::DateTime(datetime)) + } + Value::Point(Point { x, y }) => Ok(Bson::Document(doc! { "x": x, "y": y })), + Value::Inet(val) => Ok(Bson::String(val.to_string())), + Value::I16(val) => Ok(Bson::Int32(val.into())), + Value::I128(val) => Ok(Bson::Decimal128(Decimal128::from_bytes(val.to_be_bytes()))), + Value::Map(hash_map) => { + let doc = + hash_map + .into_iter() + .try_fold(Document::new(), |mut acc, (key, value)| { + acc.extend(doc! {key: value.into_bson()?}); + + Ok::<_, Error>(acc) + })?; + + Ok(Bson::Document(doc)) + } + Value::U32(val) => Ok(Bson::Int64(val.into())), + Value::U16(val) => Ok(Bson::Int32(val.into())), + Value::U128(val) => Ok(Bson::Decimal128(Decimal128::from_bytes(val.to_be_bytes()))), + Value::U8(val) => Ok(Bson::Int32(val.into())), + Value::U64(val) => { + let mut bytes_128: [u8; 16] = [0; 16]; + bytes_128[..8].copy_from_slice(&val.to_be_bytes()); + + Ok(Bson::Decimal128(Decimal128::from_bytes(bytes_128))) + } + + Value::Interval(val) => Ok(Bson::String(val.to_sql_str())), + } + } +} diff --git a/storages/mongo-storage/src/store.rs b/storages/mongo-storage/src/store.rs new file mode 100644 index 000000000..134e515bc --- /dev/null +++ b/storages/mongo-storage/src/store.rs @@ -0,0 +1,277 @@ +use { + crate::{ + error::{MongoStorageError, OptionExt, ResultExt}, + row::{key::KeyIntoBson, value::IntoValue, IntoRow}, + utils::get_primary_key, + MongoStorage, + }, + async_trait::async_trait, + futures::{stream, Stream, StreamExt, TryStreamExt}, + gluesql_core::{ + ast::{ColumnDef, ColumnUniqueOption}, + data::{Key, Schema}, + error::Result, + parse_sql::{parse_data_type, parse_expr}, + prelude::{Error, Value}, + store::{DataRow, RowIter, Store}, + translate::{translate_data_type, translate_expr}, + }, + mongodb::{ + bson::{doc, Document}, + options::{FindOptions, ListIndexesOptions}, + IndexModel, + }, + std::{collections::HashMap, future}, +}; + +#[async_trait(?Send)] +impl Store for MongoStorage { + async fn fetch_schema(&self, table_name: &str) -> Result> { + self.fetch_schemas_iter(Some(table_name)) + .await? + .next() + .await + .transpose() + } + + async fn fetch_all_schemas(&self) -> Result> { + let mut schemas = self + .fetch_schemas_iter(None) + .await? + .try_collect::>() + .await?; + + schemas.sort_by(|a, b| a.table_name.cmp(&b.table_name)); + + Ok(schemas) + } + + async fn fetch_data(&self, table_name: &str, target: &Key) -> Result> { + let column_defs = self + .get_column_defs(table_name) + .await? + .map_storage_err(MongoStorageError::Unreachable)?; + + let primary_key = get_primary_key(&column_defs) + .ok_or(MongoStorageError::Unreachable) + .map_storage_err()?; + + let filter = doc! { "_id": target.to_owned().into_bson(true)?}; + let projection = doc! {"_id": 0}; + let options = FindOptions::builder() + .projection(projection) + .sort(doc! { primary_key.name.clone(): 1 }) + .build(); + + let mut cursor = self + .db + .collection::(table_name) + .find(filter, options) + .await + .map_storage_err()?; + + cursor + .next() + .await + .transpose() + .map_storage_err()? + .map(|doc| { + doc.into_iter() + .zip(column_defs.iter()) + .map(|((_, bson), column_def)| bson.into_value(&column_def.data_type)) + .collect::>>() + .map(DataRow::Vec) + }) + .transpose() + } + + async fn scan_data(&self, table_name: &str) -> Result { + let column_defs = self.get_column_defs(table_name).await?; + + let primary_key = column_defs + .as_ref() + .and_then(|column_defs| get_primary_key(column_defs)); + + let has_primary = primary_key.is_some(); + + let options = FindOptions::builder(); + let options = match primary_key { + Some(primary_key) => options.sort(doc! { primary_key.name.to_owned(): 1}).build(), + None => options.build(), + }; + + let cursor = self + .db + .collection::(table_name) + .find(Document::new(), options) + .await + .map_storage_err()?; + + let column_types = column_defs.as_ref().map(|column_defs| { + column_defs + .iter() + .map(|column_def| column_def.data_type.clone()) + .collect::>() + }); + + let row_iter = cursor.map(move |doc| { + let doc = doc.map_storage_err()?; + + match &column_types { + Some(column_types) => doc.into_row(column_types.iter(), has_primary), + None => { + let mut iter = doc.into_iter(); + let (_, first_value) = iter + .next() + .map_storage_err(MongoStorageError::InvalidDocument)?; + let key_bytes = first_value + .as_object_id() + .map_storage_err(MongoStorageError::InvalidDocument)? + .bytes() + .to_vec(); + let key = Key::Bytea(key_bytes); + let row = iter + .map(|(key, bson)| Ok((key, bson.into_value_schemaless()?))) + .collect::>>()?; + + Ok((key, DataRow::Map(row))) + } + } + }); + + Ok(Box::pin(row_iter)) + } +} + +impl MongoStorage { + async fn fetch_schemas_iter<'a>( + &'a self, + table_name: Option<&'a str>, + ) -> Result> + 'a> { + let command = match table_name { + Some(table_name) => doc! { "listCollections": 1, "filter": { "name": table_name } }, + None => doc! { "listCollections": 1 }, + }; + + let validators_list = self + .db + .run_command(command, None) + .await + .map_storage_err()? + .get_document("cursor") + .and_then(|doc| doc.get_array("firstBatch")) + .map_storage_err()? + .to_owned(); + + let schemas = stream::iter(validators_list).then(move |validators| async move { + let doc = validators + .as_document() + .map_storage_err(MongoStorageError::InvalidDocument)?; + + let collection_name = doc.get_str("name").map_storage_err()?; + let validators = doc + .get_document("options") + .and_then(|doc| doc.get_document("validator")) + .map_storage_err()?; + + let collection = self.db.collection::(collection_name); + let options = ListIndexesOptions::builder().build(); + let cursor = collection.list_indexes(options).await.map_storage_err()?; + let indexes = cursor + .into_stream() + .map_err(|e| Error::StorageMsg(e.to_string())) + .try_filter_map(|index_model| { + let IndexModel { keys, options, .. } = index_model; + if keys.len() > 1 { + return future::ready(Ok::<_, Error>(None)); + } + + let index_keys = &mut keys.into_iter().map(|(index_key, _)| index_key); + let index_key = index_keys.next(); + let name = options.and_then(|options| options.name); + + future::ready(Ok::<_, Error>(index_key.zip(name))) + }) + .try_collect::>() + .await?; + + let column_defs = validators + .get_document("$jsonSchema") + .and_then(|doc| doc.get_document("properties")) + .map_storage_err()? + .into_iter() + .skip(1) + .map(|(column_name, doc)| { + let doc = doc + .as_document() + .map_storage_err(MongoStorageError::InvalidDocument)?; + + let nullable = doc + .get_array("bsonType") + .map_err(|_| MongoStorageError::InvalidBsonType) + .map_storage_err()? + .get(1) + .and_then(|x| x.as_str()) + .map(|x| x == "null") + .unwrap_or(false); + + let data_type = doc + .get_str("title") + .map_err(|_| MongoStorageError::InvalidGlueType) + .map_storage_err() + .and_then(parse_data_type) + .and_then(|s| translate_data_type(&s))?; + + let index_name = indexes.get(column_name).and_then(|i| i.split_once('_')); + + let unique = match index_name { + Some((_, "PK")) => Some(true), + Some((_, "UNIQUE")) => Some(false), + _ => None, + } + .map(|is_primary| ColumnUniqueOption { is_primary }); + + let default = doc + .get_str("description") + .ok() + .map(parse_expr) + .map(|expr| expr.and_then(|expr| translate_expr(&expr))) + .transpose()?; + + let column_def = ColumnDef { + name: column_name.to_owned(), + data_type, + nullable, + default, + unique, + }; + + Ok(column_def) + }) + .collect::>>()?; + + let column_defs = match column_defs.len() { + 0 => None, + _ => Some(column_defs), + }; + + let schema = Schema { + table_name: collection_name.to_owned(), + column_defs, + indexes: Vec::new(), + engine: None, + }; + + Ok::<_, Error>(schema) + }); + + Ok(Box::pin(schemas)) + } + + pub async fn get_column_defs(&self, table_name: &str) -> Result>> { + Ok(self + .fetch_schema(table_name) + .await? + .and_then(|schema| schema.column_defs)) + } +} diff --git a/storages/mongo-storage/src/store_mut.rs b/storages/mongo-storage/src/store_mut.rs new file mode 100644 index 000000000..4076cd98f --- /dev/null +++ b/storages/mongo-storage/src/store_mut.rs @@ -0,0 +1,276 @@ +use { + crate::{ + error::{MongoStorageError, OptionExt, ResultExt}, + row::{ + data_type::{BsonType, IntoRange}, + key::{into_object_id, KeyIntoBson}, + value::IntoBson, + }, + utils::{get_collection_options, get_primary_key}, + MongoStorage, + }, + async_trait::async_trait, + gluesql_core::{ + ast::{ColumnUniqueOption, ToSql}, + data::{Key, Schema}, + error::Result, + prelude::Error, + store::{DataRow, StoreMut}, + }, + mongodb::{ + bson::{doc, Bson, Document}, + options::{IndexOptions, ReplaceOptions}, + }, +}; + +struct IndexInfo { + name: String, + key: String, + index_type: IndexType, +} + +enum IndexType { + Primary, + Unique, +} + +#[async_trait(?Send)] +impl StoreMut for MongoStorage { + async fn insert_schema(&mut self, schema: &Schema) -> Result<()> { + let (labels, column_types, indexes) = schema + .column_defs + .as_ref() + .map(|column_defs| { + column_defs.iter().fold( + (Vec::new(), Document::new(), Vec::new()), + |(mut labels, mut column_types, mut indexes), column_def| { + let column_name = &column_def.name; + labels.push(column_name.clone()); + + let data_type = BsonType::from(&column_def.data_type).into(); + let maximum = column_def.data_type.get_max(); + let minimum = column_def.data_type.get_min(); + + let mut bson_type = match column_def.nullable { + true => vec![data_type, "null"], + false => vec![data_type], + }; + + match &column_def.unique { + Some(ColumnUniqueOption { is_primary }) => match *is_primary { + true => { + indexes.push(IndexInfo { + name: format!("{column_name}_PK"), + key: column_name.clone(), + index_type: IndexType::Primary, + }); + } + false => { + bson_type = vec![data_type, "null"]; + indexes.push(IndexInfo { + name: format!("{column_name}_UNIQUE"), + key: column_name.clone(), + index_type: IndexType::Unique, + }); + } + }, + None => {} + } + + let mut property = doc! { + "bsonType": bson_type, + }; + + if let Some(maximum) = maximum { + property.extend(doc! { + "maximum": maximum, + }); + } + + if let Some(minimum) = minimum { + property.extend(doc! { + "minimum": minimum, + }); + } + + if let Some(default) = &column_def.default { + property.extend(doc! { + "description": default.to_sql() + }); + } + + let type_str = column_def.data_type.to_string(); + property.extend(doc! { + "title": type_str + }); + + let column_type = doc! { + column_name: property, + }; + + column_types.extend(column_type); + + (labels, column_types, indexes) + }, + ) + }) + .unwrap_or_default(); + + let options = get_collection_options(labels, column_types); + + self.db + .create_collection(&schema.table_name, options) + .await + .map_storage_err()?; + + if indexes.is_empty() { + return Ok(()); + } + + let index_models = indexes + .into_iter() + .map( + |IndexInfo { + name, + key, + index_type, + }| { + let index_options = IndexOptions::builder().unique(true); + let index_options = match index_type { + IndexType::Primary => index_options.name(name).build(), + IndexType::Unique => index_options + .partial_filter_expression( + doc! { "partialFilterExpression": { key.clone(): { "$ne": null } } }, + ) + .name(name) + .build(), + }; + + mongodb::IndexModel::builder() + .keys(doc! {key: 1}) + .options(index_options) + .build() + }, + ) + .collect::>(); + + self.db + .collection::(&schema.table_name) + .create_indexes(index_models, None) + .await + .map(|_| ()) + .map_storage_err() + } + + async fn delete_schema(&mut self, table_name: &str) -> Result<()> { + self.db + .collection::(table_name) + .drop(None) + .await + .map_storage_err() + } + + async fn append_data(&mut self, table_name: &str, rows: Vec) -> Result<()> { + let column_defs = self.get_column_defs(table_name).await?; + + let data = rows + .into_iter() + .map(|row| match row { + DataRow::Vec(values) => column_defs + .as_ref() + .map_storage_err(MongoStorageError::Unreachable)? + .iter() + .zip(values.into_iter()) + .try_fold(Document::new(), |mut acc, (column_def, value)| { + acc.extend(doc! {column_def.name.clone(): value.into_bson()?}); + + Ok(acc) + }), + DataRow::Map(hash_map) => { + hash_map + .into_iter() + .try_fold(Document::new(), |mut acc, (key, value)| { + acc.extend(doc! {key: value.into_bson()?}); + + Ok(acc) + }) + } + }) + .collect::>>()?; + + if data.is_empty() { + return Ok(()); + } + + self.db + .collection::(table_name) + .insert_many(data, None) + .await + .map(|_| ()) + .map_storage_err() + } + + async fn insert_data(&mut self, table_name: &str, rows: Vec<(Key, DataRow)>) -> Result<()> { + let column_defs = self.get_column_defs(table_name).await?; + + let primary_key = column_defs + .as_ref() + .and_then(|column_defs| get_primary_key(column_defs)); + + for (key, row) in rows { + let doc = match row { + DataRow::Vec(values) => column_defs + .as_ref() + .map_storage_err(MongoStorageError::Unreachable)? + .iter() + .zip(values.into_iter()) + .try_fold( + doc! {"_id": key.clone().into_bson(primary_key.is_some())?}, + |mut acc, (column_def, value)| { + acc.extend(doc! {column_def.name.clone(): value.into_bson()?}); + + Ok::<_, Error>(acc) + }, + ), + DataRow::Map(hash_map) => hash_map.into_iter().try_fold( + doc! {"_id": into_object_id(key.clone())?}, + |mut acc, (key, value)| { + acc.extend(doc! {key: value.into_bson()?}); + + Ok(acc) + }, + ), + }?; + + let query = doc! {"_id": key.into_bson(primary_key.is_some())?}; + let options = ReplaceOptions::builder().upsert(Some(true)).build(); + + self.db + .collection::(table_name) + .replace_one(query, doc, options) + .await + .map_storage_err()?; + } + + Ok(()) + } + + async fn delete_data(&mut self, table_name: &str, keys: Vec) -> Result<()> { + let column_defs = self.get_column_defs(table_name).await?; + let primary_key = column_defs + .as_ref() + .and_then(|column_defs| get_primary_key(column_defs)); + + self.db + .collection::(table_name) + .delete_many( + doc! { "_id": { + "$in": keys.into_iter().map(|key| key.into_bson(primary_key.is_some())).collect::>>()? + }}, + None, + ) + .await + .map(|_| ()) + .map_storage_err() + } +} diff --git a/storages/mongo-storage/src/utils.rs b/storages/mongo-storage/src/utils.rs new file mode 100644 index 000000000..a0dcc7319 --- /dev/null +++ b/storages/mongo-storage/src/utils.rs @@ -0,0 +1,37 @@ +use { + bson::{doc, Document}, + gluesql_core::ast::ColumnDef, + mongodb::options::CreateCollectionOptions, +}; + +pub fn get_primary_key(column_defs: &[ColumnDef]) -> Option<&ColumnDef> { + column_defs + .iter() + .find(|column_def| column_def.unique.map(|x| x.is_primary).unwrap_or(false)) +} + +pub fn get_collection_options( + labels: Vec, + column_types: Document, +) -> CreateCollectionOptions { + let mut required = vec!["_id".to_owned()]; + required.extend(labels); + + let mut properties = doc! { + "_id": { "bsonType": ["objectId", "binData"] } + }; + properties.extend(column_types); + + let additional_properties = matches!(required.len(), 1); + + CreateCollectionOptions::builder() + .validator(Some(doc! { + "$jsonSchema": { + "type": "object", + "required": required, + "properties": properties, + "additionalProperties": additional_properties + } + })) + .build() +} diff --git a/storages/mongo-storage/tests/mongo_indexes.rs b/storages/mongo-storage/tests/mongo_indexes.rs new file mode 100644 index 000000000..456ed018f --- /dev/null +++ b/storages/mongo-storage/tests/mongo_indexes.rs @@ -0,0 +1,57 @@ +use { + bson::{doc, Document}, + gluesql_core::prelude::{Glue, Payload}, + gluesql_mongo_storage::{get_collection_options, MongoStorage}, + mongodb::{options::IndexOptions, IndexModel}, + std::vec, +}; + +#[tokio::test] +async fn mongo_indexes() { + let conn_str = "mongodb://localhost:27017"; + + let storage = MongoStorage::new(conn_str, "mongo_indexes") + .await + .expect("MongoStorage::new"); + storage.drop_database().await.expect("database dropped"); + + let labels = vec!["id".to_owned(), "name".to_owned()]; + let column_types = doc! { + "id": { "bsonType": ["int"], "title": "INT" }, + "name": { "bsonType": ["string"], "title": "TEXT" }, + }; + + let options = get_collection_options(labels, column_types); + + let table_name = "collection_with_composite_index"; + + storage + .db + .create_collection(table_name, options) + .await + .expect("create_collection"); + + let index_options = IndexOptions::builder() + .name("ignored_composite_index".to_owned()) + .build(); + let index_model = IndexModel::builder() + .keys(doc! {"id": 1, "name":1 }) + .options(index_options) + .build(); + let collection = storage.db.collection::(table_name); + collection.create_index(index_model, None).await.unwrap(); + + let mut glue = Glue::new(storage); + + let cases = vec![( + glue.execute(format! {"SELECT * FROM {table_name}"}).await, + Ok(Payload::Select { + labels: vec!["id".to_owned(), "name".to_owned()], + rows: vec![], + }), + )]; + + for (actual, expected) in cases { + assert_eq!(actual.map(|mut payloads| payloads.remove(0)), expected); + } +} diff --git a/storages/mongo-storage/tests/mongo_storage.rs b/storages/mongo-storage/tests/mongo_storage.rs new file mode 100644 index 000000000..da0cbb53e --- /dev/null +++ b/storages/mongo-storage/tests/mongo_storage.rs @@ -0,0 +1,29 @@ +use { + async_trait::async_trait, gluesql_core::prelude::Glue, gluesql_mongo_storage::MongoStorage, + test_suite::*, +}; + +struct MongoTester { + glue: Glue, +} + +#[async_trait(?Send)] +impl Tester for MongoTester { + async fn new(namespace: &str) -> Self { + let conn_str = "mongodb://localhost:27017"; + let storage = MongoStorage::new(conn_str, namespace) + .await + .expect("MongoStorage::new"); + storage.drop_database().await.expect("database dropped"); + let glue = Glue::new(storage); + + MongoTester { glue } + } + + fn get_glue(&mut self) -> &mut Glue { + &mut self.glue + } +} + +#[cfg(feature = "test-mongo")] +generate_store_tests!(tokio::test, MongoTester); diff --git a/storages/mongo-storage/tests/mongo_types.rs b/storages/mongo-storage/tests/mongo_types.rs new file mode 100644 index 000000000..3a2d6f824 --- /dev/null +++ b/storages/mongo-storage/tests/mongo_types.rs @@ -0,0 +1,100 @@ +use { + bson::{doc, Bson}, + gluesql_core::prelude::{Glue, Payload, Value}, + gluesql_mongo_storage::{get_collection_options, MongoStorage}, + std::{collections::HashMap, vec}, +}; + +#[tokio::test] +async fn mongo_types() { + let conn_str = "mongodb://localhost:27017"; + + let storage = MongoStorage::new(conn_str, "mongo_types") + .await + .expect("MongoStorage::new"); + storage.drop_database().await.expect("database dropped"); + + let labels = vec![ + "col_javascript".to_owned(), + "col_javascriptWithScope".to_owned(), + "col_regex".to_owned(), + "col_minKey".to_owned(), + "col_maxKey".to_owned(), + ]; + let column_types = doc! { + "col_javascript": { "bsonType": ["javascript"], "title": "TEXT" }, + "col_javascriptWithScope": { "bsonType": ["javascriptWithScope"], "title": "TEXT" }, + "col_regex": { "bsonType": ["regex"], "title": "TEXT" }, + "col_minKey": { "bsonType": ["minKey"], "title": "TEXT" }, + "col_maxKey": { "bsonType": ["maxKey"], "title": "TEXT" }, + }; + + let options = get_collection_options(labels, column_types); + + let table_name = "mongo_type_collection"; + + storage + .db + .create_collection(table_name, options) + .await + .expect("create_collection"); + + let data = doc! { + "col_javascript": Bson::JavaScriptCode("function add(a, b) { return a + b; }".to_owned()), + "col_javascriptWithScope": Bson::JavaScriptCodeWithScope(bson::JavaScriptCodeWithScope { + code: "function sub(a, b) { return a - b; }".to_owned(), + scope: doc! { "a": 1, "b": 2 } + }), + "col_regex": Bson::RegularExpression(bson::Regex { + pattern: "^[a-z]*$".to_owned(), + options: "i".to_owned() + }), + "col_minKey": Bson::MinKey, + "col_maxKey": Bson::MaxKey, + }; + + storage + .db + .collection(table_name) + .insert_one(data, None) + .await + .expect("insert_data"); + + let mut glue = Glue::new(storage); + + let cases = vec![( + glue.execute(format! {"SELECT * FROM {table_name}"}).await, + Ok(Payload::Select { + labels: vec![ + "col_javascript".to_owned(), + "col_javascriptWithScope".to_owned(), + "col_regex".to_owned(), + "col_minKey".to_owned(), + "col_maxKey".to_owned(), + ], + rows: vec![vec![ + Value::Str("function add(a, b) { return a + b; }".to_owned()), + Value::Map(HashMap::from([ + ( + "code".to_owned(), + Value::Str("function sub(a, b) { return a - b; }".to_owned()), + ), + ( + "scope".to_owned(), + Value::Map(HashMap::from([ + ("a".to_owned(), Value::I32(1)), + ("b".to_owned(), Value::I32(2)), + ])), + ), + ])), + Value::Str("/^[a-z]*$/i".to_owned()), + Value::Str("MinKey()".to_owned()), + Value::Str("MaxKey()".to_owned()), + ]], + }), + )]; + + for (actual, expected) in cases { + assert_eq!(actual.map(|mut payloads| payloads.remove(0)), expected); + } +} diff --git a/test-suite/src/function/ifnull.rs b/test-suite/src/function/ifnull.rs index 50ee1c96a..47eae517d 100644 --- a/test-suite/src/function/ifnull.rs +++ b/test-suite/src/function/ifnull.rs @@ -56,15 +56,15 @@ test_case!(ifnull, { select!("mybool" | "myfloat"; Str | Str; "YES".to_owned() "NO".to_owned()), ), ( - "SELECT IFNULL(mytime, 'YES') AS mybool, IFNULL(mytimestamp, 'NO') AS myfloat + "SELECT IFNULL(mytime, 'YES') AS mytime, IFNULL(mytimestamp, 'NO') AS mytimestamp FROM SingleItem WHERE id IS NOT NULL", - select!("mybool" | "myfloat"; Time | Timestamp; + select!("mytime" | "mytimestamp"; Time | Timestamp; NaiveTime::from_hms_opt(1, 2, 3).unwrap() NaiveDateTime::from_timestamp_opt(0, 0).unwrap()), ), ( - "SELECT IFNULL(mytime, 'YES') AS mybool, IFNULL(mytimestamp, 'NO') AS myfloat + "SELECT IFNULL(mytime, 'YES') AS mytime, IFNULL(mytimestamp, 'NO') AS mytimestamp FROM SingleItem WHERE id IS NULL", - select!("mybool" | "myfloat"; Str | Str; "YES".to_owned() "NO".to_owned()), + select!("mytime" | "mytimestamp"; Str | Str; "YES".to_owned() "NO".to_owned()), ), ]; diff --git a/test-suite/src/function/rand.rs b/test-suite/src/function/rand.rs index 5be7a6ae5..8ce2a2cdf 100644 --- a/test-suite/src/function/rand.rs +++ b/test-suite/src/function/rand.rs @@ -11,7 +11,7 @@ test_case!(rand, { let test_cases = [ ( - "CREATE TABLE SingleItem (qty INTEGER DEFAULT ROUND(RAND()*100))", + "CREATE TABLE SingleItem (qty Float DEFAULT ROUND(RAND()*100))", Ok(Payload::Create), ), (