diff --git a/Makefile b/Makefile index ffa54b3899b..0b757c6c3ab 100644 --- a/Makefile +++ b/Makefile @@ -91,7 +91,7 @@ allconfig: $(call config-cmake-release, \ -DBUILD_BENCHMARK=TRUE \ -DBUILD_EXAMPLES=TRUE \ - -DBUILD_EXTENSIONS="httpfs;duckdb;json;postgres;sqlite;fts;delta;iceberg" \ + -DBUILD_EXTENSIONS="httpfs;duckdb;json;postgres;sqlite;fts;delta;iceberg;unity_catalog" \ -DBUILD_JAVA=TRUE \ -DBUILD_NODEJS=TRUE \ -DBUILD_PYTHON=TRUE \ @@ -106,7 +106,7 @@ alldebug: $(call run-cmake-debug, \ -DBUILD_BENCHMARK=TRUE \ -DBUILD_EXAMPLES=TRUE \ - -DBUILD_EXTENSIONS="httpfs;duckdb;json;postgres;sqlite;fts;delta;iceberg" \ + -DBUILD_EXTENSIONS="httpfs;duckdb;json;postgres;sqlite;fts;delta;iceberg;unity_catalog" \ -DBUILD_JAVA=TRUE \ -DBUILD_NODEJS=TRUE \ -DBUILD_PYTHON=TRUE \ @@ -195,7 +195,7 @@ example: extension-test-build: $(call run-cmake-relwithdebinfo, \ - -DBUILD_EXTENSIONS="httpfs;duckdb;json;postgres;sqlite;fts;delta;iceberg" \ + -DBUILD_EXTENSIONS="httpfs;duckdb;json;postgres;sqlite;fts;delta;iceberg;unity_catalog" \ -DBUILD_EXTENSION_TESTS=TRUE \ -DBUILD_TESTS=TRUE \ ) @@ -221,13 +221,13 @@ extension-json-test: extension-json-test-build extension-debug: $(call run-cmake-debug, \ - -DBUILD_EXTENSIONS="httpfs;duckdb;json;postgres;sqlite;fts;delta;iceberg" \ + -DBUILD_EXTENSIONS="httpfs;duckdb;json;postgres;sqlite;fts;delta;iceberg;unity_catalog" \ -DBUILD_KUZU=FALSE \ ) extension-release: $(call run-cmake-release, \ - -DBUILD_EXTENSIONS="httpfs;duckdb;json;postgres;sqlite;fts;delta;iceberg" \ + -DBUILD_EXTENSIONS="httpfs;duckdb;json;postgres;sqlite;fts;delta;iceberg;unity_catalog" \ -DBUILD_KUZU=FALSE \ ) @@ -270,6 +270,7 @@ clean-extension: cmake -E rm -rf extension/fts/build cmake -E rm -rf extension/delta/build cmake -E rm -rf extension/iceberg/build + cmake -E rm -rf extension/unity_catalog/build clean-python-api: cmake -E rm -rf tools/python_api/build diff --git a/extension/CMakeLists.txt b/extension/CMakeLists.txt index 289bd6d9d2a..3bb7951d220 100644 --- a/extension/CMakeLists.txt +++ b/extension/CMakeLists.txt @@ -61,3 +61,7 @@ endif () if ("iceberg" IN_LIST BUILD_EXTENSIONS) add_subdirectory(iceberg) endif () + +if ("unity_catalog" IN_LIST BUILD_EXTENSIONS) + add_subdirectory(unity_catalog) +endif () diff --git a/extension/delta/src/main/CMakeLists.txt b/extension/delta/src/main/CMakeLists.txt index ccfc2b148a7..557968a69d4 100644 --- a/extension/delta/src/main/CMakeLists.txt +++ b/extension/delta/src/main/CMakeLists.txt @@ -3,11 +3,6 @@ add_library(kuzu_delta_extension delta_extension.cpp ${PROJECT_SOURCE_DIR}/extension/httpfs/src/s3_download_options.cpp) -target_include_directories( - kuzu_delta_extension - PRIVATE -) - set(DELTA_EXTENSION_OBJECT_FILES ${DELTA_EXTENSION_OBJECT_FILES} $ PARENT_SCOPE) diff --git a/extension/unity_catalog/CMakeLists.txt b/extension/unity_catalog/CMakeLists.txt new file mode 100644 index 00000000000..e0aa06dea09 --- /dev/null +++ b/extension/unity_catalog/CMakeLists.txt @@ -0,0 +1,45 @@ +if (WIN32) + set(DuckDB_USE_STATIC_LIBS ON) +else () + set(DuckDB_USE_STATIC_LIBS OFF) +endif () +find_package(DuckDB REQUIRED) + +include_directories( + ${PROJECT_SOURCE_DIR}/src/include + src/include + ${PROJECT_SOURCE_DIR}/extension/duckdb/src/include + ${DuckDB_INCLUDE_DIRS}) + +add_subdirectory(src/installer) +add_subdirectory(src/main) +add_subdirectory(src/connector) +add_subdirectory(src/storage) +add_subdirectory(src/options) + +add_library(unity_catalog_extension + SHARED + ${UNITY_CATALOG_EXTENSION_OBJECT_FILES}) + +set_extension_properties(unity_catalog_extension unity_catalog unity_catalog) + +target_link_libraries(unity_catalog_extension + PRIVATE + ${DuckDB_LIBRARIES}) + +if (NOT WIN32) + add_library(unity_catalog_loader + SHARED + ${PROJECT_SOURCE_DIR}/extension/duckdb/src/loader/duckdb_loader.cpp) + set_extension_properties(unity_catalog_loader unity_catalog_loader unity_catalog) +endif () + +if (WIN32) + # See comments in extension/httpfs/CMakeLists.txt. + target_link_libraries(unity_catalog_extension PRIVATE kuzu) +endif () + +if (APPLE) + set_apple_dynamic_lookup(unity_catalog_extension) + set_apple_dynamic_lookup(unity_catalog_loader) +endif () diff --git a/extension/unity_catalog/src/connector/CMakeLists.txt b/extension/unity_catalog/src/connector/CMakeLists.txt new file mode 100644 index 00000000000..7f16d7bef30 --- /dev/null +++ b/extension/unity_catalog/src/connector/CMakeLists.txt @@ -0,0 +1,9 @@ +add_library(kuzu_unity_catalog_connector + OBJECT + unity_catalog_connector.cpp + ${PROJECT_SOURCE_DIR}/extension/duckdb/src/connector/duckdb_connector.cpp + ${PROJECT_SOURCE_DIR}/extension/duckdb/src/connector/duckdb_result_converter.cpp) + +set(UNITY_CATALOG_EXTENSION_OBJECT_FILES + ${UNITY_CATALOG_EXTENSION_OBJECT_FILES} $ + PARENT_SCOPE) diff --git a/extension/unity_catalog/src/connector/unity_catalog_connector.cpp b/extension/unity_catalog/src/connector/unity_catalog_connector.cpp new file mode 100644 index 00000000000..12341e3e3d8 --- /dev/null +++ b/extension/unity_catalog/src/connector/unity_catalog_connector.cpp @@ -0,0 +1,24 @@ +#include "connector/unity_catalog_connector.h" + +#include "options/unity_catalog_options.h" + +namespace kuzu { +namespace unity_catalog_extension { + +void UnityCatalogConnector::connect(const std::string& dbPath, const std::string& catalogName, + const std::string& /*schemaName*/, main::ClientContext* context) { + // Creates an in-memory duckdb instance, then install httpfs and attach postgres. + instance = std::make_unique(nullptr); + connection = std::make_unique(*instance); + executeQuery("install uc_catalog from core_nightly;"); + executeQuery("load uc_catalog;"); + executeQuery("install delta;"); + executeQuery("load delta;"); + executeQuery("install delta;"); + executeQuery(DuckDBUnityCatalogSecretManager::getSecret(context)); + executeQuery(common::stringFormat("attach '{}' as {} (TYPE UC_CATALOG, read_only);", dbPath, + catalogName)); +} + +} // namespace unity_catalog_extension +} // namespace kuzu diff --git a/extension/unity_catalog/src/include/connector/unity_catalog_connector.h b/extension/unity_catalog/src/include/connector/unity_catalog_connector.h new file mode 100644 index 00000000000..047ff1a3706 --- /dev/null +++ b/extension/unity_catalog/src/include/connector/unity_catalog_connector.h @@ -0,0 +1,15 @@ +#pragma once + +#include "connector/duckdb_connector.h" + +namespace kuzu { +namespace unity_catalog_extension { + +class UnityCatalogConnector : public duckdb_extension::DuckDBConnector { +public: + void connect(const std::string& dbPath, const std::string& catalogName, + const std::string& schemaName, main::ClientContext* context) override; +}; + +} // namespace unity_catalog_extension +} // namespace kuzu diff --git a/extension/unity_catalog/src/include/main/unity_catalog_extension.h b/extension/unity_catalog/src/include/main/unity_catalog_extension.h new file mode 100644 index 00000000000..2d3b7150fd1 --- /dev/null +++ b/extension/unity_catalog/src/include/main/unity_catalog_extension.h @@ -0,0 +1,17 @@ +#pragma once + +#include "extension/extension.h" + +namespace kuzu { +namespace unity_catalog_extension { + +class UnityCatalogExtension final : public extension::Extension { +public: + static constexpr char EXTENSION_NAME[] = "UNITY_CATALOG"; + +public: + static void load(main::ClientContext* context); +}; + +} // namespace unity_catalog_extension +} // namespace kuzu diff --git a/extension/unity_catalog/src/include/options/unity_catalog_options.h b/extension/unity_catalog/src/include/options/unity_catalog_options.h new file mode 100644 index 00000000000..f5fa9c3f781 --- /dev/null +++ b/extension/unity_catalog/src/include/options/unity_catalog_options.h @@ -0,0 +1,35 @@ +#pragma once + +#include "common/types/value/value.h" + +namespace kuzu { +namespace main { +class Database; +class ClientContext; +} // namespace main + +namespace unity_catalog_extension { + +struct UnityCatalogToken { + static constexpr const char* NAME = "uc_token"; + static constexpr common::LogicalTypeID TYPE = common::LogicalTypeID::STRING; + static common::Value getDefaultValue() { return common::Value{"not-used"}; } +}; + +struct UnityCatalogEndPoint { + static constexpr const char* NAME = "uc_endpoint"; + static constexpr common::LogicalTypeID TYPE = common::LogicalTypeID::STRING; + static common::Value getDefaultValue() { return common::Value{"http://127.0.0.1:8080"}; } +}; + +struct UnityCatalogOptions { + static void registerExtensionOptions(main::Database* db); + static void setEnvValue(main::ClientContext* context); +}; + +struct DuckDBUnityCatalogSecretManager { + static std::string getSecret(main::ClientContext* context); +}; + +} // namespace unity_catalog_extension +} // namespace kuzu diff --git a/extension/unity_catalog/src/include/storage/unity_catalog_storage.h b/extension/unity_catalog/src/include/storage/unity_catalog_storage.h new file mode 100644 index 00000000000..fd951270a3f --- /dev/null +++ b/extension/unity_catalog/src/include/storage/unity_catalog_storage.h @@ -0,0 +1,24 @@ +#pragma once + +#include "storage/storage_extension.h" + +namespace kuzu { +namespace main { +class Database; +} // namespace main + +namespace unity_catalog_extension { + +class UnityCatalogStorageExtension final : public storage::StorageExtension { +public: + static constexpr const char* DB_TYPE = "UC_CATALOG"; + + static constexpr const char* DEFAULT_SCHEMA_NAME = "default"; + + explicit UnityCatalogStorageExtension(main::Database* database); + + bool canHandleDB(std::string dbType) const override; +}; + +} // namespace unity_catalog_extension +} // namespace kuzu diff --git a/extension/unity_catalog/src/installer/CMakeLists.txt b/extension/unity_catalog/src/installer/CMakeLists.txt new file mode 100644 index 00000000000..db7402a9987 --- /dev/null +++ b/extension/unity_catalog/src/installer/CMakeLists.txt @@ -0,0 +1,11 @@ +if (NOT WIN32) + add_library(unity_catalog_installer + SHARED + unity_catalog_install_func.cpp + ${PROJECT_SOURCE_DIR}/extension/duckdb/src/installer/duckdb_installer.cpp) + set_extension_properties(unity_catalog_installer unity_catalog_installer unity_catalog) +endif () + +if (APPLE) + set_apple_dynamic_lookup(unity_catalog_installer) +endif () diff --git a/extension/unity_catalog/src/installer/unity_catalog_install_func.cpp b/extension/unity_catalog/src/installer/unity_catalog_install_func.cpp new file mode 100644 index 00000000000..eed441b8a20 --- /dev/null +++ b/extension/unity_catalog/src/installer/unity_catalog_install_func.cpp @@ -0,0 +1,15 @@ +#include "installer/duckdb_installer.h" + +extern "C" { +// Because we link against the static library on windows, we implicitly inherit KUZU_STATIC_DEFINE, +// which cancels out any exporting, so we can't use KUZU_API. +#if defined(_WIN32) +#define INIT_EXPORT __declspec(dllexport) +#else +#define INIT_EXPORT __attribute__((visibility("default"))) +#endif +INIT_EXPORT void install(kuzu::main::ClientContext* context) { + kuzu::duckdb_extension::DuckDBInstaller installer{"unity_catalog"}; + installer.install(context); +} +} diff --git a/extension/unity_catalog/src/main/CMakeLists.txt b/extension/unity_catalog/src/main/CMakeLists.txt new file mode 100644 index 00000000000..0d0e61f6fcc --- /dev/null +++ b/extension/unity_catalog/src/main/CMakeLists.txt @@ -0,0 +1,7 @@ +add_library(kuzu_unity_catalog_extension + OBJECT + unity_catalog_extension.cpp) + +set(UNITY_CATALOG_EXTENSION_OBJECT_FILES + ${UNITY_CATALOG_EXTENSION_OBJECT_FILES} $ + PARENT_SCOPE) diff --git a/extension/unity_catalog/src/main/unity_catalog_extension.cpp b/extension/unity_catalog/src/main/unity_catalog_extension.cpp new file mode 100644 index 00000000000..adc99463f99 --- /dev/null +++ b/extension/unity_catalog/src/main/unity_catalog_extension.cpp @@ -0,0 +1,37 @@ + +#include "main/unity_catalog_extension.h" + +#include "main/client_context.h" +#include "options/unity_catalog_options.h" +#include "storage/unity_catalog_storage.h" + +namespace kuzu { +namespace unity_catalog_extension { + +void UnityCatalogExtension::load(main::ClientContext* context) { + auto& db = *context->getDatabase(); + db.registerStorageExtension(EXTENSION_NAME, + std::make_unique(&db)); + UnityCatalogOptions::registerExtensionOptions(&db); + UnityCatalogOptions::setEnvValue(context); +} + +} // namespace unity_catalog_extension +} // namespace kuzu + +extern "C" { +// Because we link against the static library on windows, we implicitly inherit KUZU_STATIC_DEFINE, +// which cancels out any exporting, so we can't use KUZU_API. +#if defined(_WIN32) +#define INIT_EXPORT __declspec(dllexport) +#else +#define INIT_EXPORT __attribute__((visibility("default"))) +#endif +INIT_EXPORT void init(kuzu::main::ClientContext* context) { + kuzu::unity_catalog_extension::UnityCatalogExtension::load(context); +} + +INIT_EXPORT const char* name() { + return kuzu::unity_catalog_extension::UnityCatalogExtension::EXTENSION_NAME; +} +} diff --git a/extension/unity_catalog/src/options/CMakeLists.txt b/extension/unity_catalog/src/options/CMakeLists.txt new file mode 100644 index 00000000000..c32459f6a15 --- /dev/null +++ b/extension/unity_catalog/src/options/CMakeLists.txt @@ -0,0 +1,7 @@ +add_library(kuzu_unity_catalog_options + OBJECT + unity_catalog_options.cpp) + +set(UNITY_CATALOG_EXTENSION_OBJECT_FILES + ${UNITY_CATALOG_EXTENSION_OBJECT_FILES} $ + PARENT_SCOPE) diff --git a/extension/unity_catalog/src/options/unity_catalog_options.cpp b/extension/unity_catalog/src/options/unity_catalog_options.cpp new file mode 100644 index 00000000000..a79dca095ea --- /dev/null +++ b/extension/unity_catalog/src/options/unity_catalog_options.cpp @@ -0,0 +1,48 @@ +#include "options/unity_catalog_options.h" + +#include "extension/extension.h" +#include "main/client_context.h" +#include "main/database.h" + +namespace kuzu { +namespace unity_catalog_extension { + +using namespace common; + +void UnityCatalogOptions::registerExtensionOptions(main::Database* db) { + ADD_EXTENSION_OPTION(UnityCatalogToken); + ADD_EXTENSION_OPTION(UnityCatalogEndPoint); +} + +void UnityCatalogOptions::setEnvValue(main::ClientContext* context) { + auto token = context->getEnvVariable(UnityCatalogToken::NAME); + auto endPoint = context->getEnvVariable(UnityCatalogEndPoint::NAME); + if (token != "") { + context->setExtensionOption(UnityCatalogToken::NAME, Value::createValue(token)); + } + if (endPoint != "") { + context->setExtensionOption(UnityCatalogToken::NAME, Value::createValue(endPoint)); + } +} + +static std::string getUnityCatalogOptions(main::ClientContext* context, std::string optionName) { + static common::case_insensitive_map_t UNITY_CATALOG_OPTIONS = { + {UnityCatalogToken::NAME, "TOKEN"}, {UnityCatalogEndPoint::NAME, "ENDPOINT"}}; + auto optionNameInDuckDB = UNITY_CATALOG_OPTIONS.at(optionName); + auto optionValueInKuzu = context->getCurrentSetting(optionName).toString(); + return common::stringFormat("{} '{}',", optionNameInDuckDB, optionValueInKuzu); +} + +std::string DuckDBUnityCatalogSecretManager::getSecret(main::ClientContext* context) { + std::string templateQuery = R"(CREATE SECRET ( + {} + TYPE UC + );)"; + std::string options = ""; + options += getUnityCatalogOptions(context, UnityCatalogToken::NAME); + options += getUnityCatalogOptions(context, UnityCatalogEndPoint::NAME); + return common::stringFormat(templateQuery, options); +} + +} // namespace unity_catalog_extension +} // namespace kuzu diff --git a/extension/unity_catalog/src/storage/CMakeLists.txt b/extension/unity_catalog/src/storage/CMakeLists.txt new file mode 100644 index 00000000000..dcc23d825db --- /dev/null +++ b/extension/unity_catalog/src/storage/CMakeLists.txt @@ -0,0 +1,13 @@ +add_library(kuzu_unity_catalog_storage + OBJECT + unity_catalog_storage.cpp + ${PROJECT_SOURCE_DIR}/extension/duckdb/src/catalog/duckdb_catalog.cpp + ${PROJECT_SOURCE_DIR}/extension/duckdb/src/function/duckdb_scan.cpp + ${PROJECT_SOURCE_DIR}/extension/duckdb/src/function/clear_cache.cpp + ${PROJECT_SOURCE_DIR}/extension/duckdb/src/connector/duckdb_type_converter.cpp + ${PROJECT_SOURCE_DIR}/extension/duckdb/src/catalog/duckdb_table_catalog_entry.cpp +) + +set(UNITY_CATALOG_EXTENSION_OBJECT_FILES + ${UNITY_CATALOG_EXTENSION_OBJECT_FILES} $ + PARENT_SCOPE) diff --git a/extension/unity_catalog/src/storage/unity_catalog_storage.cpp b/extension/unity_catalog/src/storage/unity_catalog_storage.cpp new file mode 100644 index 00000000000..107b09e60b4 --- /dev/null +++ b/extension/unity_catalog/src/storage/unity_catalog_storage.cpp @@ -0,0 +1,41 @@ +#include "storage/unity_catalog_storage.h" + +#include "catalog/duckdb_catalog.h" +#include "common/exception/runtime.h" +#include "common/string_utils.h" +#include "connector/unity_catalog_connector.h" +#include "extension/extension.h" +#include "function/clear_cache.h" +#include "storage/attached_duckdb_database.h" + +namespace kuzu { +namespace unity_catalog_extension { + +std::unique_ptr attachUnityCatalog(std::string dbName, std::string dbPath, + main::ClientContext* clientContext, const binder::AttachOption& attachOption) { + if (dbName == "") { + dbName = dbPath; + } + auto connector = std::make_unique(); + connector->connect(dbPath, dbName, UnityCatalogStorageExtension::DEFAULT_SCHEMA_NAME, + clientContext); + auto catalog = std::make_unique(dbPath, dbPath, + UnityCatalogStorageExtension::DEFAULT_SCHEMA_NAME, clientContext, *connector, attachOption); + catalog->init(); + return std::make_unique(dbName, + UnityCatalogStorageExtension::DB_TYPE, std::move(catalog), std::move(connector)); +} + +UnityCatalogStorageExtension::UnityCatalogStorageExtension(main::Database* database) + : StorageExtension{attachUnityCatalog} { + extension::ExtensionUtils::addStandaloneTableFunc( + *database); +} + +bool UnityCatalogStorageExtension::canHandleDB(std::string dbType_) const { + common::StringUtils::toUpper(dbType_); + return dbType_ == DB_TYPE; +} + +} // namespace unity_catalog_extension +} // namespace kuzu diff --git a/extension/unity_catalog/test/setup/create.bash b/extension/unity_catalog/test/setup/create.bash new file mode 100755 index 00000000000..9b5b9e3f3e9 --- /dev/null +++ b/extension/unity_catalog/test/setup/create.bash @@ -0,0 +1,6 @@ +${1}/bin/uc table delete --full_name university.default.grades +${1}/bin/uc table delete --full_name university.default.person +${1}/bin/uc catalog create --name university +${1}/bin/uc schema create --catalog university --name default +${1}/bin/uc table create --full_name university.default.grades --columns "name string, pass boolean, math short, chinese int, english long, physics float, chemistry double" --format DELTA --storage_location "${2}/extension/unity_catalog/test/setup/tables/grades" +${1}/bin/uc table create --full_name university.default.person --columns "name string, class byte, birthtime timestamp, info binary" --format DELTA --storage_location "${2}/extension/unity_catalog/test/setup/tables/person" diff --git a/extension/unity_catalog/test/setup/tables/grades/.part-00000-1a881821-4b10-440c-b80a-56f28db08827-c000.snappy.parquet.crc b/extension/unity_catalog/test/setup/tables/grades/.part-00000-1a881821-4b10-440c-b80a-56f28db08827-c000.snappy.parquet.crc new file mode 100644 index 00000000000..d654b4198d7 Binary files /dev/null and b/extension/unity_catalog/test/setup/tables/grades/.part-00000-1a881821-4b10-440c-b80a-56f28db08827-c000.snappy.parquet.crc differ diff --git a/extension/unity_catalog/test/setup/tables/grades/.part-00000-2cf73f64-3a60-40b0-9b44-fbfea2dd4804-c000.snappy.parquet.crc b/extension/unity_catalog/test/setup/tables/grades/.part-00000-2cf73f64-3a60-40b0-9b44-fbfea2dd4804-c000.snappy.parquet.crc new file mode 100644 index 00000000000..ae7997c77cb Binary files /dev/null and b/extension/unity_catalog/test/setup/tables/grades/.part-00000-2cf73f64-3a60-40b0-9b44-fbfea2dd4804-c000.snappy.parquet.crc differ diff --git a/extension/unity_catalog/test/setup/tables/grades/.part-00000-32af8482-113b-4786-a6f9-9e9db670a5d1-c000.snappy.parquet.crc b/extension/unity_catalog/test/setup/tables/grades/.part-00000-32af8482-113b-4786-a6f9-9e9db670a5d1-c000.snappy.parquet.crc new file mode 100644 index 00000000000..83c10883f08 Binary files /dev/null and b/extension/unity_catalog/test/setup/tables/grades/.part-00000-32af8482-113b-4786-a6f9-9e9db670a5d1-c000.snappy.parquet.crc differ diff --git a/extension/unity_catalog/test/setup/tables/grades/.part-00000-3d7b1143-211e-43a6-8d66-0173286933d6-c000.snappy.parquet.crc b/extension/unity_catalog/test/setup/tables/grades/.part-00000-3d7b1143-211e-43a6-8d66-0173286933d6-c000.snappy.parquet.crc new file mode 100644 index 00000000000..990a1fcabd9 Binary files /dev/null and b/extension/unity_catalog/test/setup/tables/grades/.part-00000-3d7b1143-211e-43a6-8d66-0173286933d6-c000.snappy.parquet.crc differ diff --git a/extension/unity_catalog/test/setup/tables/grades/.part-00000-b1000c1b-3b1f-466b-b0a6-8dda3bd8dd17-c000.snappy.parquet.crc b/extension/unity_catalog/test/setup/tables/grades/.part-00000-b1000c1b-3b1f-466b-b0a6-8dda3bd8dd17-c000.snappy.parquet.crc new file mode 100644 index 00000000000..9b644ac6712 Binary files /dev/null and b/extension/unity_catalog/test/setup/tables/grades/.part-00000-b1000c1b-3b1f-466b-b0a6-8dda3bd8dd17-c000.snappy.parquet.crc differ diff --git a/extension/unity_catalog/test/setup/tables/grades/.part-00000-d99e0d17-4aba-428b-9dbb-bb372d71859b-c000.snappy.parquet.crc b/extension/unity_catalog/test/setup/tables/grades/.part-00000-d99e0d17-4aba-428b-9dbb-bb372d71859b-c000.snappy.parquet.crc new file mode 100644 index 00000000000..ae7997c77cb Binary files /dev/null and b/extension/unity_catalog/test/setup/tables/grades/.part-00000-d99e0d17-4aba-428b-9dbb-bb372d71859b-c000.snappy.parquet.crc differ diff --git a/extension/unity_catalog/test/setup/tables/grades/.part-00000-e2a690a3-04db-4384-8b16-e6aeb0f8a722-c000.snappy.parquet.crc b/extension/unity_catalog/test/setup/tables/grades/.part-00000-e2a690a3-04db-4384-8b16-e6aeb0f8a722-c000.snappy.parquet.crc new file mode 100644 index 00000000000..467620d684c Binary files /dev/null and b/extension/unity_catalog/test/setup/tables/grades/.part-00000-e2a690a3-04db-4384-8b16-e6aeb0f8a722-c000.snappy.parquet.crc differ diff --git a/extension/unity_catalog/test/setup/tables/grades/_delta_log/.00000000000000000000.json.crc b/extension/unity_catalog/test/setup/tables/grades/_delta_log/.00000000000000000000.json.crc new file mode 100644 index 00000000000..b4125ad007f Binary files /dev/null and b/extension/unity_catalog/test/setup/tables/grades/_delta_log/.00000000000000000000.json.crc differ diff --git a/extension/unity_catalog/test/setup/tables/grades/_delta_log/.00000000000000000001.json.crc b/extension/unity_catalog/test/setup/tables/grades/_delta_log/.00000000000000000001.json.crc new file mode 100644 index 00000000000..6f4a79be63c Binary files /dev/null and b/extension/unity_catalog/test/setup/tables/grades/_delta_log/.00000000000000000001.json.crc differ diff --git a/extension/unity_catalog/test/setup/tables/grades/_delta_log/.00000000000000000002.json.crc b/extension/unity_catalog/test/setup/tables/grades/_delta_log/.00000000000000000002.json.crc new file mode 100644 index 00000000000..6e9396294c8 Binary files /dev/null and b/extension/unity_catalog/test/setup/tables/grades/_delta_log/.00000000000000000002.json.crc differ diff --git a/extension/unity_catalog/test/setup/tables/grades/_delta_log/.00000000000000000003.json.crc b/extension/unity_catalog/test/setup/tables/grades/_delta_log/.00000000000000000003.json.crc new file mode 100644 index 00000000000..08066081461 Binary files /dev/null and b/extension/unity_catalog/test/setup/tables/grades/_delta_log/.00000000000000000003.json.crc differ diff --git a/extension/unity_catalog/test/setup/tables/grades/_delta_log/.00000000000000000004.json.crc b/extension/unity_catalog/test/setup/tables/grades/_delta_log/.00000000000000000004.json.crc new file mode 100644 index 00000000000..f6dd0332ccc Binary files /dev/null and b/extension/unity_catalog/test/setup/tables/grades/_delta_log/.00000000000000000004.json.crc differ diff --git a/extension/unity_catalog/test/setup/tables/grades/_delta_log/.00000000000000000005.json.crc b/extension/unity_catalog/test/setup/tables/grades/_delta_log/.00000000000000000005.json.crc new file mode 100644 index 00000000000..3c2c11b84f5 Binary files /dev/null and b/extension/unity_catalog/test/setup/tables/grades/_delta_log/.00000000000000000005.json.crc differ diff --git a/extension/unity_catalog/test/setup/tables/grades/_delta_log/.00000000000000000006.json.crc b/extension/unity_catalog/test/setup/tables/grades/_delta_log/.00000000000000000006.json.crc new file mode 100644 index 00000000000..cd6aac9a52b Binary files /dev/null and b/extension/unity_catalog/test/setup/tables/grades/_delta_log/.00000000000000000006.json.crc differ diff --git a/extension/unity_catalog/test/setup/tables/grades/_delta_log/.00000000000000000007.json.crc b/extension/unity_catalog/test/setup/tables/grades/_delta_log/.00000000000000000007.json.crc new file mode 100644 index 00000000000..d530c4d25a5 Binary files /dev/null and b/extension/unity_catalog/test/setup/tables/grades/_delta_log/.00000000000000000007.json.crc differ diff --git a/extension/unity_catalog/test/setup/tables/grades/_delta_log/00000000000000000000.json b/extension/unity_catalog/test/setup/tables/grades/_delta_log/00000000000000000000.json new file mode 100644 index 00000000000..350a2ce5b06 --- /dev/null +++ b/extension/unity_catalog/test/setup/tables/grades/_delta_log/00000000000000000000.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1739316939665,"operation":"CREATE TABLE","operationParameters":{"partitionBy":"[]","clusterBy":"[]","description":null,"isManaged":"false","properties":"{}"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.2.1","txnId":"56c0fa7f-1430-4f90-926a-2d6ce09a416b"}} +{"metaData":{"id":"593745b3-7737-4a22-b03a-26aa2120f291","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"pass\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}},{\"name\":\"math\",\"type\":\"short\",\"nullable\":true,\"metadata\":{}},{\"name\":\"chinese\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"english\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"physics\",\"type\":\"float\",\"nullable\":true,\"metadata\":{}},{\"name\":\"chemistry\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1739316939641}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} diff --git a/extension/unity_catalog/test/setup/tables/grades/_delta_log/00000000000000000001.json b/extension/unity_catalog/test/setup/tables/grades/_delta_log/00000000000000000001.json new file mode 100644 index 00000000000..40fd45db0b1 --- /dev/null +++ b/extension/unity_catalog/test/setup/tables/grades/_delta_log/00000000000000000001.json @@ -0,0 +1,2 @@ +{"commitInfo":{"timestamp":1739316971332,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"1898"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.2.1","txnId":"3c9ffa72-ae2d-4db7-93d7-4c8fbc9c59dc"}} +{"add":{"path":"part-00000-d99e0d17-4aba-428b-9dbb-bb372d71859b-c000.snappy.parquet","partitionValues":{},"size":1898,"modificationTime":1739316971325,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"name\":\"Alice\",\"math\":5,\"chinese\":100,\"english\":120,\"physics\":50.7,\"chemistry\":78.9},\"maxValues\":{\"name\":\"Alice\",\"math\":5,\"chinese\":100,\"english\":120,\"physics\":50.7,\"chemistry\":78.9},\"nullCount\":{\"name\":0,\"pass\":0,\"math\":0,\"chinese\":0,\"english\":0,\"physics\":0,\"chemistry\":0}}"}} diff --git a/extension/unity_catalog/test/setup/tables/grades/_delta_log/00000000000000000002.json b/extension/unity_catalog/test/setup/tables/grades/_delta_log/00000000000000000002.json new file mode 100644 index 00000000000..44da6cfeb6b --- /dev/null +++ b/extension/unity_catalog/test/setup/tables/grades/_delta_log/00000000000000000002.json @@ -0,0 +1,2 @@ +{"commitInfo":{"timestamp":1739317170102,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"1898"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.2.1","txnId":"0983b691-4428-4a5a-8d7e-f6452129bc35"}} +{"add":{"path":"part-00000-2cf73f64-3a60-40b0-9b44-fbfea2dd4804-c000.snappy.parquet","partitionValues":{},"size":1898,"modificationTime":1739317170075,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"name\":\"Alice\",\"math\":5,\"chinese\":100,\"english\":120,\"physics\":50.7,\"chemistry\":78.9},\"maxValues\":{\"name\":\"Alice\",\"math\":5,\"chinese\":100,\"english\":120,\"physics\":50.7,\"chemistry\":78.9},\"nullCount\":{\"name\":0,\"pass\":0,\"math\":0,\"chinese\":0,\"english\":0,\"physics\":0,\"chemistry\":0}}"}} diff --git a/extension/unity_catalog/test/setup/tables/grades/_delta_log/00000000000000000003.json b/extension/unity_catalog/test/setup/tables/grades/_delta_log/00000000000000000003.json new file mode 100644 index 00000000000..10cdd6ae734 --- /dev/null +++ b/extension/unity_catalog/test/setup/tables/grades/_delta_log/00000000000000000003.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1739317663125,"operation":"DELETE","operationParameters":{"predicate":"[\"(name#1056 = Alice)\"]"},"readVersion":2,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"2","numRemovedBytes":"3796","numCopiedRows":"0","numDeletionVectorsAdded":"0","numDeletionVectorsRemoved":"0","numAddedChangeFiles":"0","executionTimeMs":"1022","numDeletionVectorsUpdated":"0","numDeletedRows":"2","scanTimeMs":"699","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"323"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.2.1","txnId":"1ccc8c63-a177-466e-bc89-1aceb48dcc90"}} +{"remove":{"path":"part-00000-d99e0d17-4aba-428b-9dbb-bb372d71859b-c000.snappy.parquet","deletionTimestamp":1739317663097,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1898}} +{"remove":{"path":"part-00000-2cf73f64-3a60-40b0-9b44-fbfea2dd4804-c000.snappy.parquet","deletionTimestamp":1739317663097,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1898}} diff --git a/extension/unity_catalog/test/setup/tables/grades/_delta_log/00000000000000000004.json b/extension/unity_catalog/test/setup/tables/grades/_delta_log/00000000000000000004.json new file mode 100644 index 00000000000..06061cb70ea --- /dev/null +++ b/extension/unity_catalog/test/setup/tables/grades/_delta_log/00000000000000000004.json @@ -0,0 +1,2 @@ +{"commitInfo":{"timestamp":1739317672635,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":3,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"1898"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.2.1","txnId":"c32f11a0-d3fa-46cd-88ea-d54eaf9bd3f8"}} +{"add":{"path":"part-00000-b1000c1b-3b1f-466b-b0a6-8dda3bd8dd17-c000.snappy.parquet","partitionValues":{},"size":1898,"modificationTime":1739317672629,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"name\":\"Alice\",\"math\":5,\"chinese\":100,\"english\":120,\"physics\":50.7,\"chemistry\":78.9},\"maxValues\":{\"name\":\"Alice\",\"math\":5,\"chinese\":100,\"english\":120,\"physics\":50.7,\"chemistry\":78.9},\"nullCount\":{\"name\":0,\"pass\":0,\"math\":0,\"chinese\":0,\"english\":0,\"physics\":0,\"chemistry\":0}}"}} diff --git a/extension/unity_catalog/test/setup/tables/grades/_delta_log/00000000000000000005.json b/extension/unity_catalog/test/setup/tables/grades/_delta_log/00000000000000000005.json new file mode 100644 index 00000000000..042164bab02 --- /dev/null +++ b/extension/unity_catalog/test/setup/tables/grades/_delta_log/00000000000000000005.json @@ -0,0 +1,2 @@ +{"commitInfo":{"timestamp":1739317695264,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":4,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"1884"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.2.1","txnId":"2c9d51b0-ca4d-41e8-911a-558a04fe0dca"}} +{"add":{"path":"part-00000-e2a690a3-04db-4384-8b16-e6aeb0f8a722-c000.snappy.parquet","partitionValues":{},"size":1884,"modificationTime":1739317695261,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"name\":\"Bob\",\"math\":4,\"chinese\":78,\"english\":12,\"physics\":24.5,\"chemistry\":68.7},\"maxValues\":{\"name\":\"Bob\",\"math\":4,\"chinese\":78,\"english\":12,\"physics\":24.5,\"chemistry\":68.7},\"nullCount\":{\"name\":0,\"pass\":0,\"math\":0,\"chinese\":0,\"english\":0,\"physics\":0,\"chemistry\":0}}"}} diff --git a/extension/unity_catalog/test/setup/tables/grades/_delta_log/00000000000000000006.json b/extension/unity_catalog/test/setup/tables/grades/_delta_log/00000000000000000006.json new file mode 100644 index 00000000000..ea7cfec3eff --- /dev/null +++ b/extension/unity_catalog/test/setup/tables/grades/_delta_log/00000000000000000006.json @@ -0,0 +1,2 @@ +{"commitInfo":{"timestamp":1739317714989,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":5,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"1898"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.2.1","txnId":"49e6ae3f-9bbd-4696-b9d4-06ea365d441d"}} +{"add":{"path":"part-00000-3d7b1143-211e-43a6-8d66-0173286933d6-c000.snappy.parquet","partitionValues":{},"size":1898,"modificationTime":1739317714986,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"name\":\"Carol\",\"math\":2,\"chinese\":99,\"english\":41,\"physics\":31.0,\"chemistry\":78.0},\"maxValues\":{\"name\":\"Carol\",\"math\":2,\"chinese\":99,\"english\":41,\"physics\":31.0,\"chemistry\":78.0},\"nullCount\":{\"name\":0,\"pass\":0,\"math\":0,\"chinese\":0,\"english\":0,\"physics\":0,\"chemistry\":0}}"}} diff --git a/extension/unity_catalog/test/setup/tables/grades/_delta_log/00000000000000000007.json b/extension/unity_catalog/test/setup/tables/grades/_delta_log/00000000000000000007.json new file mode 100644 index 00000000000..8728353038d --- /dev/null +++ b/extension/unity_catalog/test/setup/tables/grades/_delta_log/00000000000000000007.json @@ -0,0 +1,2 @@ +{"commitInfo":{"timestamp":1739317737286,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":6,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"1884"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.2.1","txnId":"e0f20541-5375-4dfc-8eee-a34a8fa5aa80"}} +{"add":{"path":"part-00000-1a881821-4b10-440c-b80a-56f28db08827-c000.snappy.parquet","partitionValues":{},"size":1884,"modificationTime":1739317737283,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"name\":\"Dan\",\"math\":9,\"chinese\":20,\"english\":19,\"physics\":7.7,\"chemistry\":9.2},\"maxValues\":{\"name\":\"Dan\",\"math\":9,\"chinese\":20,\"english\":19,\"physics\":7.7,\"chemistry\":9.2},\"nullCount\":{\"name\":0,\"pass\":0,\"math\":0,\"chinese\":0,\"english\":0,\"physics\":0,\"chemistry\":0}}"}} diff --git a/extension/unity_catalog/test/setup/tables/grades/part-00000-1a881821-4b10-440c-b80a-56f28db08827-c000.snappy.parquet b/extension/unity_catalog/test/setup/tables/grades/part-00000-1a881821-4b10-440c-b80a-56f28db08827-c000.snappy.parquet new file mode 100644 index 00000000000..8129993df73 Binary files /dev/null and b/extension/unity_catalog/test/setup/tables/grades/part-00000-1a881821-4b10-440c-b80a-56f28db08827-c000.snappy.parquet differ diff --git a/extension/unity_catalog/test/setup/tables/grades/part-00000-2cf73f64-3a60-40b0-9b44-fbfea2dd4804-c000.snappy.parquet b/extension/unity_catalog/test/setup/tables/grades/part-00000-2cf73f64-3a60-40b0-9b44-fbfea2dd4804-c000.snappy.parquet new file mode 100644 index 00000000000..e5da175da9d Binary files /dev/null and b/extension/unity_catalog/test/setup/tables/grades/part-00000-2cf73f64-3a60-40b0-9b44-fbfea2dd4804-c000.snappy.parquet differ diff --git a/extension/unity_catalog/test/setup/tables/grades/part-00000-32af8482-113b-4786-a6f9-9e9db670a5d1-c000.snappy.parquet b/extension/unity_catalog/test/setup/tables/grades/part-00000-32af8482-113b-4786-a6f9-9e9db670a5d1-c000.snappy.parquet new file mode 100644 index 00000000000..24f93580850 Binary files /dev/null and b/extension/unity_catalog/test/setup/tables/grades/part-00000-32af8482-113b-4786-a6f9-9e9db670a5d1-c000.snappy.parquet differ diff --git a/extension/unity_catalog/test/setup/tables/grades/part-00000-3d7b1143-211e-43a6-8d66-0173286933d6-c000.snappy.parquet b/extension/unity_catalog/test/setup/tables/grades/part-00000-3d7b1143-211e-43a6-8d66-0173286933d6-c000.snappy.parquet new file mode 100644 index 00000000000..b4e6592b8f3 Binary files /dev/null and b/extension/unity_catalog/test/setup/tables/grades/part-00000-3d7b1143-211e-43a6-8d66-0173286933d6-c000.snappy.parquet differ diff --git a/extension/unity_catalog/test/setup/tables/grades/part-00000-b1000c1b-3b1f-466b-b0a6-8dda3bd8dd17-c000.snappy.parquet b/extension/unity_catalog/test/setup/tables/grades/part-00000-b1000c1b-3b1f-466b-b0a6-8dda3bd8dd17-c000.snappy.parquet new file mode 100644 index 00000000000..0d2c5178642 Binary files /dev/null and b/extension/unity_catalog/test/setup/tables/grades/part-00000-b1000c1b-3b1f-466b-b0a6-8dda3bd8dd17-c000.snappy.parquet differ diff --git a/extension/unity_catalog/test/setup/tables/grades/part-00000-d99e0d17-4aba-428b-9dbb-bb372d71859b-c000.snappy.parquet b/extension/unity_catalog/test/setup/tables/grades/part-00000-d99e0d17-4aba-428b-9dbb-bb372d71859b-c000.snappy.parquet new file mode 100644 index 00000000000..e5da175da9d Binary files /dev/null and b/extension/unity_catalog/test/setup/tables/grades/part-00000-d99e0d17-4aba-428b-9dbb-bb372d71859b-c000.snappy.parquet differ diff --git a/extension/unity_catalog/test/setup/tables/grades/part-00000-e2a690a3-04db-4384-8b16-e6aeb0f8a722-c000.snappy.parquet b/extension/unity_catalog/test/setup/tables/grades/part-00000-e2a690a3-04db-4384-8b16-e6aeb0f8a722-c000.snappy.parquet new file mode 100644 index 00000000000..de39007885f Binary files /dev/null and b/extension/unity_catalog/test/setup/tables/grades/part-00000-e2a690a3-04db-4384-8b16-e6aeb0f8a722-c000.snappy.parquet differ diff --git a/extension/unity_catalog/test/setup/tables/person/.part-00000-5e72398b-f576-4e4b-a20c-ee39560c078c-c000.snappy.parquet.crc b/extension/unity_catalog/test/setup/tables/person/.part-00000-5e72398b-f576-4e4b-a20c-ee39560c078c-c000.snappy.parquet.crc new file mode 100644 index 00000000000..5580afcf106 Binary files /dev/null and b/extension/unity_catalog/test/setup/tables/person/.part-00000-5e72398b-f576-4e4b-a20c-ee39560c078c-c000.snappy.parquet.crc differ diff --git a/extension/unity_catalog/test/setup/tables/person/.part-00000-89006e91-5b67-4663-9ff6-cf5c6f626d56-c000.snappy.parquet.crc b/extension/unity_catalog/test/setup/tables/person/.part-00000-89006e91-5b67-4663-9ff6-cf5c6f626d56-c000.snappy.parquet.crc new file mode 100644 index 00000000000..aa700e78661 Binary files /dev/null and b/extension/unity_catalog/test/setup/tables/person/.part-00000-89006e91-5b67-4663-9ff6-cf5c6f626d56-c000.snappy.parquet.crc differ diff --git a/extension/unity_catalog/test/setup/tables/person/.part-00000-ba18ac4c-4224-4a57-b470-810ecce11744-c000.snappy.parquet.crc b/extension/unity_catalog/test/setup/tables/person/.part-00000-ba18ac4c-4224-4a57-b470-810ecce11744-c000.snappy.parquet.crc new file mode 100644 index 00000000000..69b1bf353a5 Binary files /dev/null and b/extension/unity_catalog/test/setup/tables/person/.part-00000-ba18ac4c-4224-4a57-b470-810ecce11744-c000.snappy.parquet.crc differ diff --git a/extension/unity_catalog/test/setup/tables/person/_delta_log/.00000000000000000000.json.crc b/extension/unity_catalog/test/setup/tables/person/_delta_log/.00000000000000000000.json.crc new file mode 100644 index 00000000000..ef077e33f6e Binary files /dev/null and b/extension/unity_catalog/test/setup/tables/person/_delta_log/.00000000000000000000.json.crc differ diff --git a/extension/unity_catalog/test/setup/tables/person/_delta_log/.00000000000000000001.json.crc b/extension/unity_catalog/test/setup/tables/person/_delta_log/.00000000000000000001.json.crc new file mode 100644 index 00000000000..09cd284aa28 Binary files /dev/null and b/extension/unity_catalog/test/setup/tables/person/_delta_log/.00000000000000000001.json.crc differ diff --git a/extension/unity_catalog/test/setup/tables/person/_delta_log/.00000000000000000002.json.crc b/extension/unity_catalog/test/setup/tables/person/_delta_log/.00000000000000000002.json.crc new file mode 100644 index 00000000000..4b694c82694 Binary files /dev/null and b/extension/unity_catalog/test/setup/tables/person/_delta_log/.00000000000000000002.json.crc differ diff --git a/extension/unity_catalog/test/setup/tables/person/_delta_log/.00000000000000000003.json.crc b/extension/unity_catalog/test/setup/tables/person/_delta_log/.00000000000000000003.json.crc new file mode 100644 index 00000000000..e79837d9d7d Binary files /dev/null and b/extension/unity_catalog/test/setup/tables/person/_delta_log/.00000000000000000003.json.crc differ diff --git a/extension/unity_catalog/test/setup/tables/person/_delta_log/00000000000000000000.json b/extension/unity_catalog/test/setup/tables/person/_delta_log/00000000000000000000.json new file mode 100644 index 00000000000..b36e35fd807 --- /dev/null +++ b/extension/unity_catalog/test/setup/tables/person/_delta_log/00000000000000000000.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1739376520529,"operation":"CREATE TABLE","operationParameters":{"partitionBy":"[]","clusterBy":"[]","description":null,"isManaged":"false","properties":"{}"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.2.1","txnId":"0bf19bef-85ab-494a-828b-456a8ccb6ca0"}} +{"metaData":{"id":"d8ffd89b-2dcf-4508-9101-2c2eeca66cb6","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"class\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{}},{\"name\":\"birthtime\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"info\",\"type\":\"binary\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1739376520459}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} diff --git a/extension/unity_catalog/test/setup/tables/person/_delta_log/00000000000000000001.json b/extension/unity_catalog/test/setup/tables/person/_delta_log/00000000000000000001.json new file mode 100644 index 00000000000..60aeedaef47 --- /dev/null +++ b/extension/unity_catalog/test/setup/tables/person/_delta_log/00000000000000000001.json @@ -0,0 +1,2 @@ +{"commitInfo":{"timestamp":1739376533400,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"1216"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.2.1","txnId":"f24945b1-d1de-44a9-a028-afcbd4e8406b"}} +{"add":{"path":"part-00000-ba18ac4c-4224-4a57-b470-810ecce11744-c000.snappy.parquet","partitionValues":{},"size":1216,"modificationTime":1739376533364,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"name\":\"Ben\",\"class\":1,\"birthtime\":\"2025-02-11T08:20:11.000-05:00\"},\"maxValues\":{\"name\":\"Ben\",\"class\":1,\"birthtime\":\"2025-02-11T08:20:11.000-05:00\"},\"nullCount\":{\"name\":0,\"class\":0,\"birthtime\":0,\"info\":0}}"}} diff --git a/extension/unity_catalog/test/setup/tables/person/_delta_log/00000000000000000002.json b/extension/unity_catalog/test/setup/tables/person/_delta_log/00000000000000000002.json new file mode 100644 index 00000000000..3a49296a588 --- /dev/null +++ b/extension/unity_catalog/test/setup/tables/person/_delta_log/00000000000000000002.json @@ -0,0 +1,2 @@ +{"commitInfo":{"timestamp":1739376533862,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"1222"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.2.1","txnId":"1bd5386f-ad54-48e7-843c-64ee0c6d3348"}} +{"add":{"path":"part-00000-5e72398b-f576-4e4b-a20c-ee39560c078c-c000.snappy.parquet","partitionValues":{},"size":1222,"modificationTime":1739376533854,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"name\":\"Cabin\",\"class\":3,\"birthtime\":\"2023-12-11T06:10:24.000-05:00\"},\"maxValues\":{\"name\":\"Cabin\",\"class\":3,\"birthtime\":\"2023-12-11T06:10:24.000-05:00\"},\"nullCount\":{\"name\":0,\"class\":0,\"birthtime\":0,\"info\":0}}"}} diff --git a/extension/unity_catalog/test/setup/tables/person/_delta_log/00000000000000000003.json b/extension/unity_catalog/test/setup/tables/person/_delta_log/00000000000000000003.json new file mode 100644 index 00000000000..ecdbd883c5c --- /dev/null +++ b/extension/unity_catalog/test/setup/tables/person/_delta_log/00000000000000000003.json @@ -0,0 +1,2 @@ +{"commitInfo":{"timestamp":1739376535359,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":2,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"1222"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.2.1","txnId":"acbf8e3a-aa3a-4ba2-bf5e-dbf567425298"}} +{"add":{"path":"part-00000-89006e91-5b67-4663-9ff6-cf5c6f626d56-c000.snappy.parquet","partitionValues":{},"size":1222,"modificationTime":1739376535354,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"name\":\"David\",\"class\":6,\"birthtime\":\"1988-09-11T07:20:12.000-04:00\"},\"maxValues\":{\"name\":\"David\",\"class\":6,\"birthtime\":\"1988-09-11T07:20:12.000-04:00\"},\"nullCount\":{\"name\":0,\"class\":0,\"birthtime\":0,\"info\":0}}"}} diff --git a/extension/unity_catalog/test/setup/tables/person/part-00000-5e72398b-f576-4e4b-a20c-ee39560c078c-c000.snappy.parquet b/extension/unity_catalog/test/setup/tables/person/part-00000-5e72398b-f576-4e4b-a20c-ee39560c078c-c000.snappy.parquet new file mode 100644 index 00000000000..9147d4651bc Binary files /dev/null and b/extension/unity_catalog/test/setup/tables/person/part-00000-5e72398b-f576-4e4b-a20c-ee39560c078c-c000.snappy.parquet differ diff --git a/extension/unity_catalog/test/setup/tables/person/part-00000-89006e91-5b67-4663-9ff6-cf5c6f626d56-c000.snappy.parquet b/extension/unity_catalog/test/setup/tables/person/part-00000-89006e91-5b67-4663-9ff6-cf5c6f626d56-c000.snappy.parquet new file mode 100644 index 00000000000..1213121f5ef Binary files /dev/null and b/extension/unity_catalog/test/setup/tables/person/part-00000-89006e91-5b67-4663-9ff6-cf5c6f626d56-c000.snappy.parquet differ diff --git a/extension/unity_catalog/test/setup/tables/person/part-00000-ba18ac4c-4224-4a57-b470-810ecce11744-c000.snappy.parquet b/extension/unity_catalog/test/setup/tables/person/part-00000-ba18ac4c-4224-4a57-b470-810ecce11744-c000.snappy.parquet new file mode 100644 index 00000000000..dc896061a26 Binary files /dev/null and b/extension/unity_catalog/test/setup/tables/person/part-00000-ba18ac4c-4224-4a57-b470-810ecce11744-c000.snappy.parquet differ diff --git a/extension/unity_catalog/test/test_files/unity_catalog.test b/extension/unity_catalog/test/test_files/unity_catalog.test new file mode 100644 index 00000000000..9b61120f694 --- /dev/null +++ b/extension/unity_catalog/test/test_files/unity_catalog.test @@ -0,0 +1,23 @@ +-DATASET CSV empty + +-- + +-CASE ScanUnityCatalog +-SKIP +-STATEMENT load extension "${KUZU_ROOT_DIRECTORY}/extension/unity_catalog/build/libunity_catalog.kuzu_extension" +---- ok +-STATEMENT load extension "${KUZU_ROOT_DIRECTORY}/extension/unity_catalog/build/libunity_catalog.kuzu_extension" +---- ok + +-LOG UniversityCatalogScan +-STATEMENT ATTACH 'university' AS university (DBTYPE UC_CATALOG); +---- ok +-STATEMENT LOAD FROM university.grades RETURN *; +---- 4 +Alice|True|5|100|120|50.700001|78.900000 +Bob|False|4|78|12|24.500000|68.700000 +Carol|True|2|99|41|31.000000|78.000000 +Dan|False|9|20|19|7.700000|9.200000 +# Duckdb has an issue when scanning this table, enable after duckdb fixes it. +#-STATEMENT LOAD FROM university.person RETURN *; +#---- 1 diff --git a/src/include/extension/extension.h b/src/include/extension/extension.h index 062f1e6672f..59d4d828bb4 100644 --- a/src/include/extension/extension.h +++ b/src/include/extension/extension.h @@ -65,7 +65,7 @@ struct KUZU_API ExtensionUtils { static constexpr const char* EXTENSION_FILE_NAME = "lib{}.kuzu_extension"; static constexpr const char* OFFICIAL_EXTENSION[] = {"HTTPFS", "POSTGRES", "DUCKDB", "JSON", - "SQLITE", "FTS", "DELTA", "ICEBERG"}; + "SQLITE", "FTS", "DELTA", "ICEBERG", "UNITY_CATALOG"}; static constexpr const char* EXTENSION_LOADER_SUFFIX = "_loader";