diff --git a/release-notes/2.2.0.md b/release-notes/2.2.0.md index 4841e464..a78d42ff 100644 --- a/release-notes/2.2.0.md +++ b/release-notes/2.2.0.md @@ -15,7 +15,7 @@ ## Library Changes and Additions -- Added `mysql-event-listener`, `session-property-manager`, and +- Added `clickhouse`, `mysql-event-listener`, `session-property-manager`, and `resource-groups` modules. - Added named volume to `insights` module to persist Postgres data. - Changed all `/etc/starburst/` references to `etc/` in properties files. diff --git a/src/lib/minitrino.env b/src/lib/minitrino.env index ae9516e4..1cbd5ef0 100644 --- a/src/lib/minitrino.env +++ b/src/lib/minitrino.env @@ -1,5 +1,6 @@ COMPOSE_PROJECT_NAME=minitrino +CLICKHOUSE_VER=23.10-alpine CURL_VER=8.4.0 DB2_VER=11.5.8.0 ELASTICSEARCH_VER=8.11.0 diff --git a/src/lib/modules/catalog/clickhouse/clickhouse.yaml b/src/lib/modules/catalog/clickhouse/clickhouse.yaml new file mode 100644 index 00000000..00276240 --- /dev/null +++ b/src/lib/modules/catalog/clickhouse/clickhouse.yaml @@ -0,0 +1,19 @@ +--- +services: + + trino: + volumes: + - ./modules/catalog/clickhouse/resources/trino/clickhouse.properties:${ETC}/catalog/clickhouse.properties + + clickhouse: + image: clickhouse/clickhouse-server:${CLICKHOUSE_VER} + container_name: clickhouse + env_file: + - ./modules/catalog/clickhouse/resources/clickhouse/clickhouse.env + ports: + - :8123 + volumes: + - ./modules/catalog/clickhouse/resources/clickhouse/init.sh:/docker-entrypoint-initdb.d/init.sh + labels: + - com.starburst.tests=minitrino + - com.starburst.tests.module.clickhouse=catalog-clickhouse diff --git a/src/lib/modules/catalog/clickhouse/metadata.json b/src/lib/modules/catalog/clickhouse/metadata.json new file mode 100644 index 00000000..0ae68e04 --- /dev/null +++ b/src/lib/modules/catalog/clickhouse/metadata.json @@ -0,0 +1,5 @@ +{ + "description": "ClickHouse catalog module", + "incompatibleModules": [], + "enterprise": false +} \ No newline at end of file diff --git a/src/lib/modules/catalog/clickhouse/readme.md b/src/lib/modules/catalog/clickhouse/readme.md new file mode 100644 index 00000000..9388e5d5 --- /dev/null +++ b/src/lib/modules/catalog/clickhouse/readme.md @@ -0,0 +1,24 @@ +# ClickHouse Catalog Module + +This module provisions a ClickHouse server with some preloaded data. The +preloaded tables are stored in the `minitrino` ClickHouse database, which is +exposed as the `clickhouse.minitrino` schema in Trino. + +## Loading Data + +Data is loaded by mounting a shell script to the ClickHouse container's +`docker-entrypoint-initdb.d/` directory. Scripts in this directory are executed +whenever the container boots up, and is the ClickHouse-supported method for +bootstrapping ClickHouse containers. + +The init script can be edited in the library to load different tables and/or +additional data: + + lib/modules/catalog/clickhouse/resources/clickhouse/init.sh + +## Usage + + minitrino --env STARBURST_VER= provision --module clickhouse + docker exec -it trino bash + trino-cli + trino> show tables in clickhouse.minitrino; diff --git a/src/lib/modules/catalog/clickhouse/resources/clickhouse/clickhouse.env b/src/lib/modules/catalog/clickhouse/resources/clickhouse/clickhouse.env new file mode 100644 index 00000000..57e9b87b --- /dev/null +++ b/src/lib/modules/catalog/clickhouse/resources/clickhouse/clickhouse.env @@ -0,0 +1,4 @@ +CLICKHOUSE_DB=minitrino +CLICKHOUSE_USER=admin +CLICKHOUSE_PASSWORD=trinoRocks15 +CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 diff --git a/src/lib/modules/catalog/clickhouse/resources/clickhouse/init.sh b/src/lib/modules/catalog/clickhouse/resources/clickhouse/init.sh new file mode 100755 index 00000000..3f47e29f --- /dev/null +++ b/src/lib/modules/catalog/clickhouse/resources/clickhouse/init.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +set -e + +echo "Creating three sample tables..." + +clickhouse client -n <<-EOSQL + CREATE TABLE IF NOT EXISTS minitrino.table1 ( + id UInt32, + name String, + value Float64 + ) ENGINE = MergeTree() + ORDER BY id; + + CREATE TABLE IF NOT EXISTS minitrino.table2 ( + id UInt32, + category String, + amount Decimal(10, 2) + ) ENGINE = MergeTree() + ORDER BY id; + + CREATE TABLE IF NOT EXISTS minitrino.table3 ( + id UInt32, + timestamp DateTime, + is_active UInt8 + ) ENGINE = MergeTree() + ORDER BY id; + + -- Insert random data into table1 + INSERT INTO minitrino.table1 + SELECT + number AS id, + concat('Name_', toString(number % 100)) AS name, + rand() % 10000 / 100.0 AS value + FROM numbers(1000); + + -- Insert random data into table2 + INSERT INTO minitrino.table2 + SELECT + number AS id, + concat('Category_', toString(rand() % 10)) AS category, + rand() % 5000 / 100.0 AS amount + FROM numbers(1000); + + -- Insert random data into table3 + INSERT INTO minitrino.table3 + SELECT + number AS id, + now() - number * 60 AS timestamp, + rand() % 2 AS is_active + FROM numbers(1000); +EOSQL diff --git a/src/lib/modules/catalog/clickhouse/resources/trino/clickhouse.properties b/src/lib/modules/catalog/clickhouse/resources/trino/clickhouse.properties new file mode 100644 index 00000000..9c92d4b4 --- /dev/null +++ b/src/lib/modules/catalog/clickhouse/resources/trino/clickhouse.properties @@ -0,0 +1,5 @@ +connector.name=clickhouse +connection-url=jdbc:clickhouse://clickhouse:8123 +connection-user=admin +connection-password=trinoRocks15 +case-insensitive-name-matching=true diff --git a/src/test/src/lib/json/clickhouse.json b/src/test/src/lib/json/clickhouse.json new file mode 100644 index 00000000..7f6ac2a8 --- /dev/null +++ b/src/test/src/lib/json/clickhouse.json @@ -0,0 +1,70 @@ +{ + "tests": [ + { + "type": "query", + "name": "Test create schema", + "sql": "CREATE SCHEMA clickhouse.test_schema", + "trinoCliArgs": [ + "--server 'http://trino:8080'", + "--user admin" + ], + "contains": [ + "CREATE SCHEMA" + ] + }, + { + "type": "query", + "name": "Test schema list", + "sql": "SHOW SCHEMAS IN clickhouse", + "trinoCliArgs": [ + "--server 'http://trino:8080'", + "--user admin" + ], + "contains": [ + "information_schema", + "test_schema", + "minitrino" + ] + }, + { + "type": "query", + "name": "Test table list", + "sql": "SHOW TABLES IN clickhouse.minitrino", + "trinoCliArgs": [ + "--server 'http://trino:8080'", + "--user admin" + ], + "contains": [ + "table1", + "table2", + "table3" + ] + }, + { + "type": "query", + "name": "Test CTAS", + "sql": "CREATE TABLE clickhouse.test_schema.test_table AS SELECT * FROM tpch.tiny.customer LIMIT 10", + "trinoCliArgs": [ + "--server 'http://trino:8080'", + "--user admin" + ], + "contains": [ + "CREATE TABLE: 10 rows" + ] + }, + { + "type": "query", + "name": "Test table select", + "sql": "SELECT * FROM clickhouse.test_schema.test_table LIMIT 10", + "trinoCliArgs": [ + "--server 'http://trino:8080'", + "--user admin" + ], + "contains": [ + "custkey", + "name", + "address" + ] + } + ] +}