diff --git a/readme.md b/readme.md index 4c4ded16..1e2d7d53 100644 --- a/readme.md +++ b/readme.md @@ -18,7 +18,7 @@ Slack](https://img.shields.io/static/v1?logo=slack&logoColor=959DA5&label=Slack& ----- -**Latest Stable Release**: 2.2.2 +**Latest Stable Release**: 2.2.1 ----- diff --git a/release-notes/2.2.2.md b/release-notes/2.2.2.md deleted file mode 100644 index c714d01a..00000000 --- a/release-notes/2.2.2.md +++ /dev/null @@ -1,29 +0,0 @@ -# Minitrino Release Notes: 2.2.2 - -## Release Overview - -- [Minitrino Release Notes: 2.2.2](#minitrino-release-notes-222) - - [Release Overview](#release-overview) - - [CLI Changes and Additions](#cli-changes-and-additions) - - [Library Changes and Additions](#library-changes-and-additions) - - [Other](#other) - -## CLI Changes and Additions - -- Removed enormous and unsightly environment variable string preceding `docker - compose up` commands. It is no longer required since all Minitrino and shell - environment variables are sourced and provided to the subprocess that executes - the command. -- The `remove` command now logs removed images and volumes at the `info` level, - meaning `-v` is no longer a required argument to see which objects are - removed. -- Added full command output for commands that fail to error log. - -## Library Changes and Additions - -- Added named volumes for data persistence to all catalog modules (except `Db2`) - and modified the relevant bootstrap scripts to be idempotent. - -## Other - -- N/A diff --git a/src/cli/minitrino/cmd/cmd_provision.py b/src/cli/minitrino/cmd/cmd_provision.py index f1832bff..ababee44 100644 --- a/src/cli/minitrino/cmd/cmd_provision.py +++ b/src/cli/minitrino/cmd/cmd_provision.py @@ -112,7 +112,7 @@ def cli(ctx, modules, workers, no_rollback, docker_native): cmd_chunk = chunk(modules) compose_cmd = build_command(docker_native, cmd_chunk) - ctx.cmd_executor.execute_commands(compose_cmd, environment=ctx.env.copy()) + ctx.cmd_executor.execute_commands(compose_cmd, environment=ctx.env) c_restart = execute_bootstraps(modules) c_restart = write_trino_cfg(c_restart, modules) @@ -253,8 +253,14 @@ def build_command(ctx, docker_native="", chunk=""): docker compose command string.""" cmd = [] + compose_env_string = "" + for k, v in ctx.env.items(): + compose_env_string += f'{k.upper()}="{v}" ' + cmd.extend( [ + compose_env_string, + "\\\n", "docker compose -f ", os.path.join(ctx.minitrino_lib_dir, "docker-compose.yaml"), " \\\n", diff --git a/src/cli/minitrino/cmd/cmd_remove.py b/src/cli/minitrino/cmd/cmd_remove.py index 8897f308..1d1c8515 100644 --- a/src/cli/minitrino/cmd/cmd_remove.py +++ b/src/cli/minitrino/cmd/cmd_remove.py @@ -107,7 +107,7 @@ def remove_items(ctx, item_type, force, labels=[]): ) else: ctx.docker_client.images.remove(image.short_id) - ctx.logger.info( + ctx.logger.verbose( f"{item_type.title()} removed: {identifier}", ) except APIError as e: @@ -124,7 +124,7 @@ def remove_items(ctx, item_type, force, labels=[]): volume.remove(force=True) else: volume.remove() - ctx.logger.info( + ctx.logger.verbose( f"{item_type.title()} removed: {identifier}", ) except APIError as e: diff --git a/src/cli/minitrino/components.py b/src/cli/minitrino/components.py index a4b6ea2d..8d7eb508 100644 --- a/src/cli/minitrino/components.py +++ b/src/cli/minitrino/components.py @@ -141,7 +141,7 @@ def _user_init(self, verbose=False, user_env=[]): self.env._log_env_vars() self.cmd_executor = CommandExecutor(self) - self._get_docker_clients(env=self.env.copy()) + self._get_docker_clients(env=self.env) def _handle_minitrino_user_dir(self): """Checks if a Minitrino directory exists in the user home directory. @@ -543,8 +543,7 @@ def _execute_in_shell(self, command="", **kwargs): if process.returncode != 0 and kwargs.get("trigger_error", True): raise err.MinitrinoError( f"Failed to execute shell command:\n{command}\n" - f"Exit code: {process.returncode}\n" - f"Command output: {self._strip_ansi(output)}" + f"Exit code: {process.returncode}" ) return { @@ -621,16 +620,13 @@ def _execute_in_container(self, command="", **kwargs): self._ctx.logger.warn( f"The command exited with a 126 code which typically means an " f"executable is not accessible or installed. Does this image have " - f"all required dependencies installed?\n" - f"Command: {command}\n" - f"Command output: {output}" + f"all required dependencies installed?\nCommand: {command}", ) if exit_code != 0 and kwargs.get("trigger_error", True): raise err.MinitrinoError( f"Failed to execute command in container '{container.name}':\n{command}\n" - f"Exit code: {exit_code}\n" - f"Command output: {output}" + f"Exit code: {exit_code}" ) return {"command": command, "output": output, "exit_code": exit_code} diff --git a/src/cli/setup.py b/src/cli/setup.py index 955eeb80..7feb671d 100644 --- a/src/cli/setup.py +++ b/src/cli/setup.py @@ -10,7 +10,7 @@ setup( name="minitrino", - version="2.2.2", + version="2.2.1", description="A command line tool that makes it easy to run modular Trino environments locally.", long_description=README, long_description_content_type="text/markdown", diff --git a/src/lib/modules/catalog/clickhouse/clickhouse.yaml b/src/lib/modules/catalog/clickhouse/clickhouse.yaml index 85b2c827..00276240 100644 --- a/src/lib/modules/catalog/clickhouse/clickhouse.yaml +++ b/src/lib/modules/catalog/clickhouse/clickhouse.yaml @@ -13,14 +13,7 @@ services: ports: - :8123 volumes: - - clickhouse-data:/var/lib/clickhouse - ./modules/catalog/clickhouse/resources/clickhouse/init.sh:/docker-entrypoint-initdb.d/init.sh labels: - com.starburst.tests=minitrino - com.starburst.tests.module.clickhouse=catalog-clickhouse - -volumes: - clickhouse-data: - labels: - - com.starburst.tests=minitrino - - com.starburst.tests.module.clickhouse=catalog-clickhouse diff --git a/src/lib/modules/catalog/clickhouse/readme.md b/src/lib/modules/catalog/clickhouse/readme.md index bef703f0..6c7620d5 100644 --- a/src/lib/modules/catalog/clickhouse/readme.md +++ b/src/lib/modules/catalog/clickhouse/readme.md @@ -30,36 +30,3 @@ trino-cli trino> SHOW TABLES IN clickhouse.minitrino; ``` - -## Persistent Storage - -This module uses named volumes to persist ClickHouse data: - -```yaml -volumes: - clickhouse-data: - labels: - - com.starburst.tests=minitrino - - com.starburst.tests.module.clickhouse=catalog-clickhouse -``` - -The user-facing implication is that ClickHouse data is retained even after -shutting down and/or removing the environment's containers. Minitrino issues a -warning about this whenever a module with named volumes is deployed––be sure to -look out for these warnings: - -```log -[w] Module '' has persistent volumes associated with it. To delete these volumes, remember to run `minitrino remove --volumes`. -``` - -To remove these volumes, run: - -```sh -minitrino -v remove --volumes --label com.starburst.tests.module.clickhouse=catalog-clickhouse -``` - -Or, remove them directly using the Docker CLI: - -```sh -docker volume rm minitrino_clickhouse-data -``` diff --git a/src/lib/modules/catalog/clickhouse/resources/clickhouse/init.sh b/src/lib/modules/catalog/clickhouse/resources/clickhouse/init.sh index e1662946..3f47e29f 100755 --- a/src/lib/modules/catalog/clickhouse/resources/clickhouse/init.sh +++ b/src/lib/modules/catalog/clickhouse/resources/clickhouse/init.sh @@ -26,30 +26,27 @@ clickhouse client -n <<-EOSQL ) ENGINE = MergeTree() ORDER BY id; - -- Insert random data into table1 only if there are fewer than 1000 rows + -- Insert random data into table1 INSERT INTO minitrino.table1 SELECT number AS id, concat('Name_', toString(number % 100)) AS name, rand() % 10000 / 100.0 AS value - FROM numbers(1000) - WHERE (SELECT count() FROM minitrino.table1) < 1000; + FROM numbers(1000); - -- Insert random data into table2 only if there are fewer than 1000 rows + -- Insert random data into table2 INSERT INTO minitrino.table2 SELECT number AS id, concat('Category_', toString(rand() % 10)) AS category, rand() % 5000 / 100.0 AS amount - FROM numbers(1000) - WHERE (SELECT count() FROM minitrino.table2) < 1000; + FROM numbers(1000); - -- Insert random data into table3 only if there are fewer than 1000 rows + -- Insert random data into table3 INSERT INTO minitrino.table3 SELECT number AS id, now() - number * 60 AS timestamp, rand() % 2 AS is_active - FROM numbers(1000) - WHERE (SELECT count() FROM minitrino.table3) < 1000; + FROM numbers(1000); EOSQL diff --git a/src/lib/modules/catalog/elasticsearch/elasticsearch.yaml b/src/lib/modules/catalog/elasticsearch/elasticsearch.yaml index d1063323..27dc4e87 100644 --- a/src/lib/modules/catalog/elasticsearch/elasticsearch.yaml +++ b/src/lib/modules/catalog/elasticsearch/elasticsearch.yaml @@ -20,14 +20,6 @@ services: ports: - 9200:9200 - 9300:9300 - volumes: - - elasticsearch-data:/usr/share/elasticsearch/data - labels: - - com.starburst.tests=minitrino - - com.starburst.tests.module.elasticsearch=catalog-elasticsearch - -volumes: - elasticsearch-data: labels: - com.starburst.tests=minitrino - com.starburst.tests.module.elasticsearch=catalog-elasticsearch diff --git a/src/lib/modules/catalog/elasticsearch/readme.md b/src/lib/modules/catalog/elasticsearch/readme.md index d548f598..0170d62e 100644 --- a/src/lib/modules/catalog/elasticsearch/readme.md +++ b/src/lib/modules/catalog/elasticsearch/readme.md @@ -74,36 +74,3 @@ by this module, located at: ```sh lib/modules/catalog/elasticsearch/resources/bootstrap/bootstrap-elasticsearch.sh ``` - -## Persistent Storage - -This module uses named volumes to persist ES data: - -```yaml -volumes: - elasticsearch-data: - labels: - - com.starburst.tests=minitrino - - com.starburst.tests.module.elasticsearch=catalog-elasticsearch -``` - -The user-facing implication is that ES data is retained even after shutting down -and/or removing the environment's containers. Minitrino issues a warning about -this whenever a module with named volumes is deployed––be sure to look out for -these warnings: - -```log -[w] Module '' has persistent volumes associated with it. To delete these volumes, remember to run `minitrino remove --volumes`. -``` - -To remove these volumes, run: - -```sh -minitrino -v remove --volumes --label com.starburst.tests.module.elasticsearch=catalog-elasticsearch -``` - -Or, remove them directly using the Docker CLI: - -```sh -docker volume rm minitrino_elasticsearch-data -``` diff --git a/src/lib/modules/catalog/elasticsearch/resources/bootstrap/bootstrap-elasticsearch.sh b/src/lib/modules/catalog/elasticsearch/resources/bootstrap/bootstrap-elasticsearch.sh index 6cb0652d..bd3d0ebb 100755 --- a/src/lib/modules/catalog/elasticsearch/resources/bootstrap/bootstrap-elasticsearch.sh +++ b/src/lib/modules/catalog/elasticsearch/resources/bootstrap/bootstrap-elasticsearch.sh @@ -1,39 +1,38 @@ #!/usr/bin/env bash +#----------------------------------------------------------------------------------------------- +# https://github.com/andrewpuch/elasticsearch_examples +#----------------------------------------------------------------------------------------------- + set -euxo pipefail echo "Waiting for Elasticsearch to come up..." wait-for-it elasticsearch:9200 --strict --timeout=60 -- echo "Elasticsearch service is up." -echo "Checking if 'user' index exists..." -if curl -s http://elasticsearch:9200/_cat/indices | grep -q 'user'; then - echo "'user' index already exists. Skipping index creation and mapping." -else - echo "Creating 'user' index..." - curl -XPUT -H 'Content-Type: application/json' http://elasticsearch:9200/user?pretty=true -d' - { - "settings" : { - "index" : { - "number_of_replicas" : 0 - } +echo "Creating user index..." +curl -XPUT -H 'Content-Type: application/json' http://elasticsearch:9200/user?pretty=true -d' +{ + "settings" : { + "index" : { + "number_of_replicas" : 0 } - }' + } +}' - echo "Creating 'user' mapping..." - curl -XPUT 'http://elasticsearch:9200/user/_mapping' -H 'Content-Type: application/json' -d ' - { - "properties" : { - "full_name" : { "type" : "text", "store" : true }, - "bio" : { "type" : "text", "store" : true }, - "age" : { "type" : "integer" }, - "location" : { "type" : "text" }, - "enjoys_coffee" : { "type" : "boolean" }, - "created_on" : { "type" : "date" } - } - }' -fi +echo "Creating user mapping..." +curl -XPUT 'http://elasticsearch:9200/user/_mapping' -H 'Content-Type: application/json' -d ' +{ + "properties" : { + "full_name" : { "type" : "text", "store" : true }, + "bio" : { "type" : "text", "store" : true }, + "age" : { "type" : "integer" }, + "location" : { "type" : "text" }, + "enjoys_coffee" : { "type" : "boolean" }, + "created_on" : { "type" : "date" } + } +} +'; -echo "Generating and inserting sample data..." sudo pip install faker requests cat << EOF > /tmp/generate_es_users.py @@ -54,14 +53,12 @@ for i in range(1, 500): } response = requests.post( - f"http://elasticsearch:9200/user/_doc/{i}", + f"http://elasticsearch:9200/user/_doc/{i}?pretty=true", headers={"Content-Type": "application/json"}, data=json.dumps(user) ) - if response.status_code == 409: - print(f"User {i} already exists. Skipping.") - else: - print(f"Created user {i}, response: {response.status_code}") + + print(f"Created user {i}, response: {response.status_code}") EOF # Make the Python script executable diff --git a/src/lib/modules/catalog/iceberg/iceberg.yaml b/src/lib/modules/catalog/iceberg/iceberg.yaml index 0b3080e2..d1fe1c3a 100644 --- a/src/lib/modules/catalog/iceberg/iceberg.yaml +++ b/src/lib/modules/catalog/iceberg/iceberg.yaml @@ -26,14 +26,11 @@ services: AWS_ACCESS_KEY_ID: access-key AWS_SECRET_ACCESS_KEY: secret-key AWS_REGION: us-east-1 - CATALOG_URI: jdbc:sqlite:/home/iceberg/iceberg.db CATALOG_WAREHOUSE: s3://sample-bucket/wh/ CATALOG_IO__IMPL: org.apache.iceberg.aws.s3.S3FileIO CATALOG_S3_ENDPOINT: http://s3.us-east-1.minio.com:9000 ports: - 8181:8181 - volumes: - - iceberg-metadata:/home/iceberg labels: - com.starburst.tests=minitrino - com.starburst.tests.module.iceberg=catalog-iceberg @@ -92,10 +89,6 @@ services: - com.starburst.tests.module.iceberg=catalog-iceberg volumes: - iceberg-metadata: - labels: - - com.starburst.tests=minitrino - - com.starburst.tests.module.iceberg=catalog-iceberg minio-iceberg-data: labels: - com.starburst.tests=minitrino diff --git a/src/lib/modules/catalog/iceberg/readme.md b/src/lib/modules/catalog/iceberg/readme.md index 513414c9..dfd181d2 100644 --- a/src/lib/modules/catalog/iceberg/readme.md +++ b/src/lib/modules/catalog/iceberg/readme.md @@ -28,24 +28,20 @@ CREATE TABLE iceberg.test.test_tbl AS SELECT * FROM tpch.tiny.customer; ## Persistent Storage -This module uses named volumes to persist Iceberg metadata and MinIO data: +This module uses named volumes to persist MinIO data: ```yaml volumes: - iceberg-metadata: - labels: - - com.starburst.tests=minitrino - - com.starburst.tests.module.iceberg=catalog-iceberg minio-iceberg-data: labels: - com.starburst.tests=minitrino - com.starburst.tests.module.iceberg=catalog-iceberg ``` -The user-facing implication is that the Iceberg's metadata and the data files -stored in MinIO are retained even after shutting down and/or removing the -environment's containers. Minitrino issues a warning about this whenever a -module with named volumes is deployed––be sure to look out for these warnings: +The user-facing implication is that the data files stored in MinIO are retained +even after shutting down and/or removing the environment's containers. Minitrino +issues a warning about this whenever a module with named volumes is deployed––be +sure to look out for these warnings: ```log [w] Module '' has persistent volumes associated with it. To delete these volumes, remember to run `minitrino remove --volumes`. @@ -60,7 +56,7 @@ minitrino -v remove --volumes --label com.starburst.tests.module.iceberg=catalog Or, remove them directly using the Docker CLI: ```sh -docker volume rm minitrino_iceberg-metadata minitrino_minio-iceberg-data +docker volume rm minitrino_minio-iceberg-data ``` ## Editing the `iceberg.properties` File diff --git a/src/lib/modules/catalog/mariadb/mariadb.yaml b/src/lib/modules/catalog/mariadb/mariadb.yaml index 99eda373..c7cc881d 100644 --- a/src/lib/modules/catalog/mariadb/mariadb.yaml +++ b/src/lib/modules/catalog/mariadb/mariadb.yaml @@ -14,14 +14,6 @@ services: - ./modules/catalog/mariadb/resources/mariadb/mariadb.env ports: - :3306 - volumes: - - mariadb-data:/var/lib/mysql - labels: - - com.starburst.tests=minitrino - - com.starburst.tests.module.mariadb=catalog-mariadb - -volumes: - mariadb-data: labels: - com.starburst.tests=minitrino - com.starburst.tests.module.mariadb=catalog-mariadb diff --git a/src/lib/modules/catalog/mariadb/readme.md b/src/lib/modules/catalog/mariadb/readme.md index 814d8f1d..1e344259 100644 --- a/src/lib/modules/catalog/mariadb/readme.md +++ b/src/lib/modules/catalog/mariadb/readme.md @@ -31,36 +31,3 @@ trino-cli trino> SHOW SCHEMAS FROM mariadb; ``` - -## Persistent Storage - -This module uses named volumes to persist MariaDB data: - -```yaml -volumes: - mariadb-data: - labels: - - com.starburst.tests=minitrino - - com.starburst.tests.module.mariadb=catalog-mariadb -``` - -The user-facing implication is that MariaDB data is retained even after shutting -down and/or removing the environment's containers. Minitrino issues a warning -about this whenever a module with named volumes is deployed––be sure to look out -for these warnings: - -```log -[w] Module '' has persistent volumes associated with it. To delete these volumes, remember to run `minitrino remove --volumes`. -``` - -To remove these volumes, run: - -```sh -minitrino -v remove --volumes --label com.starburst.tests.module.mariadb=catalog-mariadb -``` - -Or, remove them directly using the Docker CLI: - -```sh -docker volume rm minitrino_mariadb-data -``` diff --git a/src/lib/modules/catalog/mariadb/resources/bootstrap/bootstrap-mariadb.sh b/src/lib/modules/catalog/mariadb/resources/bootstrap/bootstrap-mariadb.sh index ec717d29..c15b6a4f 100755 --- a/src/lib/modules/catalog/mariadb/resources/bootstrap/bootstrap-mariadb.sh +++ b/src/lib/modules/catalog/mariadb/resources/bootstrap/bootstrap-mariadb.sh @@ -3,14 +3,6 @@ MAX_ATTEMPTS=60 for (( ATTEMPTS=1; ATTEMPTS<=MAX_ATTEMPTS; ATTEMPTS++ )) do - # Check if the admin user already has privileges - mariadb -ptrinoRocks15 -e "SHOW GRANTS FOR 'admin';" | grep -q "GRANT ALL PRIVILEGES ON *.* TO 'admin'" - if [ $? == 0 ]; then - echo "Privileges already granted to 'admin'. Skipping privilege grant." - break - fi - - # Grant privileges if they are not already granted mariadb -ptrinoRocks15 -e "GRANT ALL PRIVILEGES ON *.* TO 'admin';" if [ $? == 0 ]; then break @@ -18,5 +10,5 @@ do sleep 1 done -echo "Ensuring 'minitrino' database exists..." -mariadb -ptrinoRocks15 -e "CREATE DATABASE IF NOT EXISTS minitrino;" +echo "Creating 'minitrino' database..." +mariadb -ptrinoRocks15 -e "CREATE DATABASE minitrino;" diff --git a/src/lib/modules/catalog/mysql/mysql.yaml b/src/lib/modules/catalog/mysql/mysql.yaml index f8089b0a..2f3e3845 100644 --- a/src/lib/modules/catalog/mysql/mysql.yaml +++ b/src/lib/modules/catalog/mysql/mysql.yaml @@ -14,14 +14,6 @@ services: - ./modules/catalog/mysql/resources/mysql/mysql.env ports: - 3306:3306 - volumes: - - mysql-data:/var/lib/mysql - labels: - - com.starburst.tests=minitrino - - com.starburst.tests.module.mysql=catalog-mysql - -volumes: - mysql-data: labels: - com.starburst.tests=minitrino - com.starburst.tests.module.mysql=catalog-mysql diff --git a/src/lib/modules/catalog/mysql/readme.md b/src/lib/modules/catalog/mysql/readme.md index 8009e420..7793cd52 100644 --- a/src/lib/modules/catalog/mysql/readme.md +++ b/src/lib/modules/catalog/mysql/readme.md @@ -22,36 +22,3 @@ docker exec -it trino bash trino-cli trino> SHOW SCHEMAS FROM mysql; ``` - -## Persistent Storage - -This module uses named volumes to persist MySQL data: - -```yaml -volumes: - mysql-data: - labels: - - com.starburst.tests=minitrino - - com.starburst.tests.module.mysql=catalog-mysql -``` - -The user-facing implication is that MySQL data is retained even after shutting -down and/or removing the environment's containers. Minitrino issues a warning -about this whenever a module with named volumes is deployed––be sure to look out -for these warnings: - -```log -[w] Module '' has persistent volumes associated with it. To delete these volumes, remember to run `minitrino remove --volumes`. -``` - -To remove these volumes, run: - -```sh -minitrino -v remove --volumes --label com.starburst.tests.module.mysql=catalog-mysql -``` - -Or, remove them directly using the Docker CLI: - -```sh -docker volume rm minitrino_mysql-data -``` diff --git a/src/lib/modules/catalog/mysql/resources/bootstrap/bootstrap-mysql.sh b/src/lib/modules/catalog/mysql/resources/bootstrap/bootstrap-mysql.sh index dd82476f..f92b970b 100755 --- a/src/lib/modules/catalog/mysql/resources/bootstrap/bootstrap-mysql.sh +++ b/src/lib/modules/catalog/mysql/resources/bootstrap/bootstrap-mysql.sh @@ -3,14 +3,6 @@ MAX_ATTEMPTS=60 for (( ATTEMPTS=1; ATTEMPTS<=MAX_ATTEMPTS; ATTEMPTS++ )) do - # Check if the admin user already has privileges - mysql -ptrinoRocks15 -e "SHOW GRANTS FOR 'admin';" | grep -q "GRANT ALL PRIVILEGES ON *.* TO 'admin'" - if [ $? == 0 ]; then - echo "Privileges already granted to 'admin'. Skipping privilege grant." - break - fi - - # Grant privileges if they are not already granted mysql -ptrinoRocks15 -e "GRANT ALL PRIVILEGES ON *.* TO 'admin';" if [ $? == 0 ]; then break @@ -18,5 +10,5 @@ do sleep 1 done -echo "Ensuring 'minitrino' database exists..." -mysql -ptrinoRocks15 -e "CREATE DATABASE IF NOT EXISTS minitrino;" +echo "Creating 'minitrino' database..." +mysql -ptrinoRocks15 -e "CREATE DATABASE minitrino;" diff --git a/src/lib/modules/catalog/pinot/pinot.yaml b/src/lib/modules/catalog/pinot/pinot.yaml index e9ebbae6..58288c98 100644 --- a/src/lib/modules/catalog/pinot/pinot.yaml +++ b/src/lib/modules/catalog/pinot/pinot.yaml @@ -13,9 +13,6 @@ services: ZOOKEEPER_TICK_TIME: 2000 ports: - :2181 - volumes: - - pinot-zookeeper-data:/data - - pinot-zookeeper-datalog:/datalog labels: - com.starburst.tests=minitrino - com.starburst.tests.module.pinot=catalog-pinot @@ -33,8 +30,6 @@ services: - pinot-zookeeper ports: - 9090:9000 # localhost:9090 - volumes: - - pinot-controller-data:/tmp/data/PinotController labels: - com.starburst.tests=minitrino - com.starburst.tests.module.pinot=catalog-pinot @@ -65,26 +60,6 @@ services: - pinot-broker ports: - :8098 - volumes: - - pinot-server-data:/tmp/data/pinotServerData - labels: - - com.starburst.tests=minitrino - - com.starburst.tests.module.pinot=catalog-pinot - -volumes: - pinot-zookeeper-data: - labels: - - com.starburst.tests=minitrino - - com.starburst.tests.module.pinot=catalog-pinot - pinot-zookeeper-datalog: - labels: - - com.starburst.tests=minitrino - - com.starburst.tests.module.pinot=catalog-pinot - pinot-controller-data: - labels: - - com.starburst.tests=minitrino - - com.starburst.tests.module.pinot=catalog-pinot - pinot-server-data: labels: - com.starburst.tests=minitrino - com.starburst.tests.module.pinot=catalog-pinot diff --git a/src/lib/modules/catalog/pinot/readme.md b/src/lib/modules/catalog/pinot/readme.md index 53e23254..a356d6a3 100644 --- a/src/lib/modules/catalog/pinot/readme.md +++ b/src/lib/modules/catalog/pinot/readme.md @@ -27,53 +27,3 @@ trino> SHOW SCHEMAS FROM pinot; ``` The Pinot web UI can be viewed at `localhost:9090`. - -## Persistent Storage - -This module uses named volumes to persist Zookeeper, Pinot controller, and Pinot -server data: - -```yaml -volumes: - pinot-zookeeper-data: - labels: - - com.starburst.tests=minitrino - - com.starburst.tests.module.pinot=catalog-pinot - pinot-zookeeper-datalog: - labels: - - com.starburst.tests=minitrino - - com.starburst.tests.module.pinot=catalog-pinot - pinot-controller-data: - labels: - - com.starburst.tests=minitrino - - com.starburst.tests.module.pinot=catalog-pinot - pinot-server-data: - labels: - - com.starburst.tests=minitrino - - com.starburst.tests.module.pinot=catalog-pinot -``` - -The user-facing implication is that the data stored in Zookeeper as well as the -Pinot components are retained even after shutting down and/or removing the -environment's containers. Minitrino issues a warning about this whenever a -module with named volumes is deployed––be sure to look out for these warnings: - -```log -[w] Module '' has persistent volumes associated with it. To delete these volumes, remember to run `minitrino remove --volumes`. -``` - -To remove these volumes, run: - -```sh -minitrino -v remove --volumes --label com.starburst.tests.module.pinot=catalog-pinot -``` - -Or, remove them directly using the Docker CLI: - -```sh -docker volume rm \ - pinot-zookeeper-data \ - pinot-zookeeper-datalog \ - minitrino_pinot-controller-data \ - minitrino_pinot-server-data -``` diff --git a/src/lib/modules/catalog/pinot/resources/bootstrap/bootstrap-pinot.sh b/src/lib/modules/catalog/pinot/resources/bootstrap/bootstrap-pinot.sh index 3825171c..530f181b 100755 --- a/src/lib/modules/catalog/pinot/resources/bootstrap/bootstrap-pinot.sh +++ b/src/lib/modules/catalog/pinot/resources/bootstrap/bootstrap-pinot.sh @@ -5,7 +5,7 @@ set -euxo pipefail echo "Checking component statuses..." check_health() { local url=$1 - local retries=36 + local retries=20 local wait=5 for ((i=0; i SHOW SCHEMAS FROM postgres; ``` - -## Persistent Storage - -This module uses named volumes to persist Postgres data: - -```yaml -volumes: - postgres-data: - labels: - - com.starburst.tests=minitrino - - com.starburst.tests.module.postgres=catalog-postgres -``` - -The user-facing implication is that Postgres data is retained even after -shutting down and/or removing the environment's containers. Minitrino issues a -warning about this whenever a module with named volumes is deployed––be sure to -look out for these warnings: - -```log -[w] Module '' has persistent volumes associated with it. To delete these volumes, remember to run `minitrino remove --volumes`. -``` - -To remove these volumes, run: - -```sh -minitrino -v remove --volumes --label com.starburst.tests.module.postgres=catalog-postgres -``` - -Or, remove them directly using the Docker CLI: - -```sh -docker volume rm minitrino_postgres-data -``` diff --git a/src/lib/modules/catalog/sqlserver/readme.md b/src/lib/modules/catalog/sqlserver/readme.md index b21ee495..42e5b606 100644 --- a/src/lib/modules/catalog/sqlserver/readme.md +++ b/src/lib/modules/catalog/sqlserver/readme.md @@ -23,36 +23,3 @@ trino-cli trino> SHOW SCHEMAS FROM sqlserver; ``` - -## Persistent Storage - -This module uses named volumes to persist SQL Server data: - -```yaml -volumes: - sqlserver-data: - labels: - - com.starburst.tests=minitrino - - com.starburst.tests.module.sqlserver=catalog-sqlserver -``` - -The user-facing implication is that SQL Server data is retained even after -shutting down and/or removing the environment's containers. Minitrino issues a -warning about this whenever a module with named volumes is deployed––be sure to -look out for these warnings: - -```log -[w] Module '' has persistent volumes associated with it. To delete these volumes, remember to run `minitrino remove --volumes`. -``` - -To remove these volumes, run: - -```sh -minitrino -v remove --volumes --label com.starburst.tests.module.sqlserver=catalog-sqlserver -``` - -Or, remove them directly using the Docker CLI: - -```sh -docker volume rm minitrino_sqlserver-data -``` diff --git a/src/lib/modules/catalog/sqlserver/sqlserver.yaml b/src/lib/modules/catalog/sqlserver/sqlserver.yaml index c5675ed2..61f6cb95 100644 --- a/src/lib/modules/catalog/sqlserver/sqlserver.yaml +++ b/src/lib/modules/catalog/sqlserver/sqlserver.yaml @@ -12,14 +12,6 @@ services: - ./modules/catalog/sqlserver/resources/sqlserver/sqlserver.env ports: - 1433:1433 - volumes: - - sqlserver-data:/var/opt/mssql - labels: - - com.starburst.tests=minitrino - - com.starburst.tests.module.sqlserver=catalog-sqlserver - -volumes: - sqlserver-data: labels: - com.starburst.tests=minitrino - com.starburst.tests.module.sqlserver=catalog-sqlserver diff --git a/src/lib/modules/catalog/test/resources/bootstrap/bootstrap-test.sh b/src/lib/modules/catalog/test/resources/bootstrap/bootstrap-test.sh index 18830be4..30848dd0 100755 --- a/src/lib/modules/catalog/test/resources/bootstrap/bootstrap-test.sh +++ b/src/lib/modules/catalog/test/resources/bootstrap/bootstrap-test.sh @@ -1,4 +1,4 @@ #!/usr/bin/env bash set -euxo pipefail -echo "hello world" > /tmp/bootstrap.txt +echo "hello world" > ~/test_bootstrap.txt diff --git a/src/lib/modules/catalog/test/resources/bootstrap/bootstrap-trino.sh b/src/lib/modules/catalog/test/resources/bootstrap/bootstrap-trino.sh index 18830be4..c7999af1 100755 --- a/src/lib/modules/catalog/test/resources/bootstrap/bootstrap-trino.sh +++ b/src/lib/modules/catalog/test/resources/bootstrap/bootstrap-trino.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash set -euxo pipefail -echo "hello world" > /tmp/bootstrap.txt + +touch /etc/starburst/test_bootstrap.txt diff --git a/src/lib/modules/catalog/test/test.yaml b/src/lib/modules/catalog/test/test.yaml index af16da7c..8d062bac 100644 --- a/src/lib/modules/catalog/test/test.yaml +++ b/src/lib/modules/catalog/test/test.yaml @@ -2,9 +2,14 @@ services: trino: - labels: - - com.starburst.tests=minitrino - - com.starburst.tests.module.test=catalog-test + environment: + MINITRINO_BOOTSTRAP: bootstrap-trino.sh + CONFIG_PROPERTIES: |- + query.max-stage-count=85 + query.max-execution-time=1h + JVM_CONFIG: |- + -Xmx2G + -Xms1G test: build: @@ -12,6 +17,8 @@ services: labels: - com.starburst.tests=minitrino - com.starburst.tests.module.test=catalog-test + environment: + MINITRINO_BOOTSTRAP: bootstrap-test.sh command: [tail, -f, /dev/null] # Keep alive image: minitrino/test:latest container_name: test diff --git a/src/lib/version b/src/lib/version index b1b25a5f..c043eea7 100644 --- a/src/lib/version +++ b/src/lib/version @@ -1 +1 @@ -2.2.2 +2.2.1 diff --git a/src/test/setup.py b/src/test/setup.py index bc94bc5e..0f4a714f 100644 --- a/src/test/setup.py +++ b/src/test/setup.py @@ -7,5 +7,5 @@ name="minitrino-tests", version="0.0", packages=["src", "src/cli", "src/lib"], - install_requires=["minitrino", "jsonschema", "PyYAML", "requests>=2.32.2"], + install_requires=["minitrino", "jsonschema", "requests>=2.32.2"], ) diff --git a/src/test/src/cli/test_cmd_provision.py b/src/test/src/cli/test_cmd_provision.py index f151d57f..b0925b9c 100644 --- a/src/test/src/cli/test_cmd_provision.py +++ b/src/test/src/cli/test_cmd_provision.py @@ -1,8 +1,10 @@ #!usr/bin/env/python3 # -*- coding: utf-8 -*- +# TODO: Test no rollback +# TODO: Test invalid user config (Trino/JVM) + import docker -import yaml import src.common as common import src.cli.utils as utils @@ -22,7 +24,6 @@ def main(): test_bad_sep_version() test_invalid_module() test_docker_native() - test_bootstrap() test_enterprise() test_version_requirements() test_valid_user_config() @@ -94,7 +95,9 @@ def test_invalid_module(): def test_docker_native(): """Ensures that native Docker Compose command options can be appended to the - provisioning command.""" + provisioning command. + + This function also calls the bootstrap script test functions.""" common.log_status(cast(FrameType, currentframe()).f_code.co_name) @@ -107,6 +110,8 @@ def test_docker_native(): assert len(containers) == 2 common.log_success(cast(FrameType, currentframe()).f_code.co_name) + + test_bootstrap_script(result) cleanup() @@ -115,7 +120,7 @@ def test_enterprise(): common.log_status(cast(FrameType, currentframe()).f_code.co_name) - utils.update_metadata_json("test", [{"enterprise": True}]) + utils.update_metadata_json([{"enterprise": True}]) result = utils.execute_cli_cmd(["-v", "provision", "--module", "test"]) assert result.exit_code == 2 @@ -138,15 +143,15 @@ def test_enterprise(): ) assert "LIC_PATH" and "/tmp/dummy.license" in result.output - utils.reset_test_metadata_json() + utils.reset_metadata_json() cleanup() # Ensure default dummy license satisfies Compose env vars result = utils.execute_cli_cmd(["-v", "provision", "--module", "test"]) assert result.exit_code == 0 - etc_ls = common.execute_command(f"docker exec -i trino ls /etc/starburst/") - assert "dummy.license" in etc_ls["output"] + assert "LIC_PATH" and "./modules/resources/dummy.license" in result.output + assert "LIC_MOUNT_PATH" and "/etc/starburst/dummy.license:ro" in result.output common.log_success(cast(FrameType, currentframe()).f_code.co_name) cleanup() @@ -157,7 +162,7 @@ def test_version_requirements(): common.log_status(cast(FrameType, currentframe()).f_code.co_name) - utils.update_metadata_json("test", [{"versions": [MIN_SEP_VER + 1, 998]}]) + utils.update_metadata_json([{"versions": [MIN_SEP_VER + 1, 998]}]) # Should fail - lower bound result = utils.execute_cli_cmd( @@ -184,79 +189,33 @@ def test_version_requirements(): assert "maximum required" in result.output common.log_success(cast(FrameType, currentframe()).f_code.co_name) - utils.reset_test_metadata_json() + utils.reset_metadata_json() cleanup() -def test_bootstrap(): +def test_bootstrap_script(result): """Ensures that bootstrap scripts properly execute in containers.""" common.log_status(cast(FrameType, currentframe()).f_code.co_name) - def add_yaml_bootstrap(yaml_path=""): - # In case left over from previous run - del_yaml_bootstrap(yaml_path) - - with open(yaml_path, "r") as file: - data = yaml.safe_load(file) - - for svc_name, svc_content in data.get("services", {}).items(): - if "environment" not in svc_content: - svc_content["environment"] = { - "MINITRINO_BOOTSTRAP": f"bootstrap-{svc_name}.sh" - } - - with open(yaml_path, "w") as file: - yaml.dump(data, file, default_flow_style=False) - - def del_yaml_bootstrap(yaml_path=""): - with open(yaml_path, "r") as file: - data = yaml.safe_load(file) - - try: - for _, svc_content in data.get("services", {}).items(): - if "environment" in svc_content: - del svc_content["environment"] - with open(yaml_path, "w") as file: - yaml.dump(data, file, default_flow_style=False) - except: - pass - - yaml_path = utils.get_module_yaml_path("test") - add_yaml_bootstrap(yaml_path) - - result = utils.execute_cli_cmd( - [ - "-v", - "provision", - "--module", - "test", - ] - ) - assert all( ( - "Successfully executed bootstrap script in container 'trino'" - in result.output, - "Successfully executed bootstrap script in container 'test'" - in result.output, + "Successfully executed bootstrap script in container: 'trino'", + "Successfully executed bootstrap script in container: 'test'", ) ) - trino_bootstrap = common.execute_command( - f"docker exec -i trino cat /tmp/bootstrap.txt" + trino_bootstrap_check = common.execute_command( + f"docker exec -i trino ls /etc/starburst/" ) - test_bootstrap = common.execute_command( - f"docker exec -i test cat /tmp/bootstrap.txt" + test_bootstrap_check = common.execute_command( + f"docker exec -i test cat /root/test_bootstrap.txt" ) - assert "hello world" in trino_bootstrap.get("output", "") and test_bootstrap.get( - "output", "" - ) + assert "test_bootstrap.txt" in trino_bootstrap_check.get("output", "") + assert "hello world" in test_bootstrap_check.get("output", "") common.log_success(cast(FrameType, currentframe()).f_code.co_name) - del_yaml_bootstrap(yaml_path) - cleanup() def test_valid_user_config(): @@ -268,10 +227,6 @@ def test_valid_user_config(): result = utils.execute_cli_cmd( [ "-v", - "--env", - "CONFIG_PROPERTIES=query.max-stage-count=85\nquery.max-execution-time=1h", - "--env", - "JVM_CONFIG=-Xms1G\n-Xmx2G", "provision", "--module", "test", diff --git a/src/test/src/cli/utils.py b/src/test/src/cli/utils.py index 1fd0e616..b171a4c6 100644 --- a/src/test/src/cli/utils.py +++ b/src/test/src/cli/utils.py @@ -61,11 +61,11 @@ def make_sample_config(): common.execute_command(cmd) -def update_metadata_json(module="", updates=[]): - """Updates a given module's metadata.json file with the provided +def update_metadata_json(updates=[]): + """Updates the test module's metadata.json file with the provided list of dicts.""" - path = get_metadata_json_path(module) + path = get_metadata_json("test") with open(path, "r") as f: data = json.load(f) @@ -78,7 +78,7 @@ def update_metadata_json(module="", updates=[]): json.dump(data, f, indent=4) -def reset_test_metadata_json(): +def reset_metadata_json(): """Resets the test module's metadata.json file to default values.""" default = { @@ -88,31 +88,16 @@ def reset_test_metadata_json(): "versions": [], "enterprise": False, } - path = get_metadata_json_path("test") + path = get_metadata_json("test") with open(path, "w") as f: - json.dump(default, f, indent=2) - f.write("\n") + json.dump(default, f, indent=4) -def get_metadata_json_path(module=""): +def get_metadata_json(module=""): """Fetches the metadata.json file path for a given module.""" - metadata = get_module_metadata(module) - return os.path.abspath( - os.path.join(metadata[module]["module_dir"], "metadata.json") - ) - - -def get_module_metadata(module=""): - """Fetches (all) module metadata for a given module.""" - - metadata = execute_cli_cmd(["modules", "-m", module, "--json"]) - return json.loads(metadata.output) - - -def get_module_yaml_path(module=""): - """Fetches the module.yaml file path for a given module.""" + result = execute_cli_cmd(["modules", "-m", module, "--json"]) + output = json.loads(result.output) - metadata = get_module_metadata(module) - return os.path.abspath(metadata[module]["yaml_file"]) + return os.path.join(output[module]["module_dir"], "metadata.json")