From d1b84c501b56e6e05542cd4147f11b240ec1c788 Mon Sep 17 00:00:00 2001 From: Noritaka Sekiyama Date: Mon, 13 Nov 2023 20:55:15 +0900 Subject: [PATCH 1/5] Reorganizing commits --- tests/functional/adapter/test_basic.py | 54 ++++++++++----------- tests/functional/adapter/test_docs.py | 11 ++++- tests/functional/adapter/test_snapshot.py | 57 ++++------------------- 3 files changed, 45 insertions(+), 77 deletions(-) diff --git a/tests/functional/adapter/test_basic.py b/tests/functional/adapter/test_basic.py index 4c64aec6..3fc926c7 100644 --- a/tests/functional/adapter/test_basic.py +++ b/tests/functional/adapter/test_basic.py @@ -1,35 +1,23 @@ -import pytest - import os + +import pytest +from dbt.tests.adapter.basic.files import (base_ephemeral_sql, base_table_sql, + base_view_sql, ephemeral_table_sql, + ephemeral_view_sql) +from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod from dbt.tests.adapter.basic.test_base import BaseSimpleMaterializations -from dbt.tests.adapter.basic.test_singular_tests import BaseSingularTests -from dbt.tests.adapter.basic.test_singular_tests_ephemeral import BaseSingularTestsEphemeral from dbt.tests.adapter.basic.test_empty import BaseEmpty from dbt.tests.adapter.basic.test_ephemeral import BaseEphemeral -from dbt.tests.adapter.basic.test_incremental import BaseIncremental from dbt.tests.adapter.basic.test_generic_tests import BaseGenericTests -from dbt.tests.adapter.basic.test_docs_generate import BaseDocsGenerate, BaseDocsGenReferences -from dbt.tests.adapter.basic.test_snapshot_check_cols import BaseSnapshotCheckCols -from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp -from dbt.tests.adapter.basic.files import ( - base_view_sql, - base_table_sql, - base_ephemeral_sql, - ephemeral_view_sql, - ephemeral_table_sql -) - -from dbt.tests.util import ( - run_dbt, - get_manifest, - check_result_nodes_by_name, - relation_from_name, - check_relations_equal, -) - +from dbt.tests.adapter.basic.test_incremental import BaseIncremental +from dbt.tests.adapter.basic.test_singular_tests import BaseSingularTests +from dbt.tests.adapter.basic.test_singular_tests_ephemeral import BaseSingularTestsEphemeral +from dbt.tests.adapter.basic.test_table_materialization import BaseTableMaterialization +from dbt.tests.adapter.basic.test_validate_connection import BaseValidateConnection +from dbt.tests.util import (check_relations_equal, check_result_nodes_by_name, + get_manifest, relation_from_name, run_dbt) from tests.util import get_s3_location, get_region, cleanup_s3_location - s3bucket = get_s3_location() region = get_region() schema_name = "dbt_functional_test_01" @@ -59,6 +47,10 @@ base_materialized_var_sql = config_materialized_var + config_incremental_strategy + model_base +class TestBaseCachingGlue(BaseAdapterMethod): + pass + + class TestSimpleMaterializationsGlue(BaseSimpleMaterializations): # all tests within this test has the same schema @pytest.fixture(scope="class") @@ -167,6 +159,7 @@ def test_ephemeral(self, project): pass + class TestSingularTestsEphemeralGlue(BaseSingularTestsEphemeral): @pytest.fixture(scope="class") def unique_schema(request, prefix) -> str: @@ -174,6 +167,7 @@ def unique_schema(request, prefix) -> str: pass + class TestIncrementalGlue(BaseIncremental): @pytest.fixture(scope='class', autouse=True) def cleanup(self): @@ -267,4 +261,12 @@ def test_generic_tests(self, project): results = run_dbt(["test"]) assert len(results) == 3 - pass \ No newline at end of file + pass + + +class TestTableMatGlue(BaseTableMaterialization): + pass + + +class TestValidateConnectionGlue(BaseValidateConnection): + pass diff --git a/tests/functional/adapter/test_docs.py b/tests/functional/adapter/test_docs.py index 3d72e52d..e8a86f20 100644 --- a/tests/functional/adapter/test_docs.py +++ b/tests/functional/adapter/test_docs.py @@ -1,9 +1,9 @@ import pytest -from dbt.tests.adapter.basic.test_docs_generate import BaseDocsGenerate +from dbt.tests.adapter.basic.test_docs_generate import (BaseDocsGenerate, + BaseDocsGenReferences) from dbt.tests.adapter.basic.expected_catalog import no_stats - schema_name = "dbt_functional_test_docs01" @@ -107,3 +107,10 @@ def case(x): pass + +class TestDocsGenReferencesGlue(BaseDocsGenReferences): + @pytest.fixture(scope="class") + def unique_schema(request, prefix) -> str: + return schema_name + + pass diff --git a/tests/functional/adapter/test_snapshot.py b/tests/functional/adapter/test_snapshot.py index 0d93dd6d..8abcff64 100644 --- a/tests/functional/adapter/test_snapshot.py +++ b/tests/functional/adapter/test_snapshot.py @@ -1,11 +1,10 @@ import pytest from dbt.tests.adapter.basic.test_snapshot_check_cols import BaseSnapshotCheckCols +from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp from dbt.tests.util import run_dbt, relation_from_name - from tests.util import get_s3_location, get_region, cleanup_s3_location - s3bucket = get_s3_location() region = get_region() schema_name = "dbt_functional_test_snapshot01" @@ -48,7 +47,7 @@ def project_config_update(self): } def test_snapshot_check_cols(self, project): - # seed command + # seed commandte results = run_dbt(["seed"]) assert len(results) == 2 @@ -76,52 +75,12 @@ def test_snapshot_check_cols(self, project): check_relation_rows(project, "cc_name_snapshot", 20) check_relation_rows(project, "cc_date_snapshot", 20) - # # update some timestamps in the "added" seed so the snapshot sees 10 more new rows - # update_rows_config = { - # "name": "added", - # "dst_col": "some_date", - # "clause": {"src_col": "some_date", "type": "add_timestamp"}, - # "where": "id > 10 and id < 21", - # } - # update_rows(project.adapter, update_rows_config) - # - # # re-run snapshots, using "added' - # results = run_dbt(["snapshot", "--vars", "seed_name: added"]) - # for result in results: - # assert result.status == "success" - # - # # check rowcounts for all snapshots - # check_relation_rows(project, "cc_all_snapshot", 30) - # check_relation_rows(project, "cc_date_snapshot", 30) - # # unchanged: only the timestamp changed - # check_relation_rows(project, "cc_name_snapshot", 20) - # - # # Update the name column - # update_rows_config = { - # "name": "added", - # "dst_col": "name", - # "clause": { - # "src_col": "name", - # "type": "add_string", - # "value": "_updated", - # }, - # "where": "id < 11", - # } - # update_rows(project.adapter, update_rows_config) - # - # # re-run snapshots, using "added' - # results = run_dbt(["snapshot", "--vars", "seed_name: added"]) - # for result in results: - # assert result.status == "success" - # - # # check rowcounts for all snapshots - # check_relation_rows(project, "cc_all_snapshot", 40) - # check_relation_rows(project, "cc_name_snapshot", 30) - # # does not see name updates - # check_relation_rows(project, "cc_date_snapshot", 30) - pass -#class TestSnapshotTimestampGlue(BaseSnapshotTimestamp): -# pass \ No newline at end of file +class TestSnapshotTimestampGlue(BaseSnapshotTimestamp): + @pytest.fixture(scope="class") + def unique_schema(request, prefix) -> str: + return schema_name + + pass From 6886039324cb8f86864d3578637dfe9e7e97bd4c Mon Sep 17 00:00:00 2001 From: Noritaka Sekiyama Date: Tue, 14 Nov 2023 15:10:49 +0900 Subject: [PATCH 2/5] Added TestSnapshotTimestampGlue --- tests/functional/adapter/test_snapshot.py | 44 ++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/tests/functional/adapter/test_snapshot.py b/tests/functional/adapter/test_snapshot.py index 8abcff64..e4601a75 100644 --- a/tests/functional/adapter/test_snapshot.py +++ b/tests/functional/adapter/test_snapshot.py @@ -7,7 +7,7 @@ s3bucket = get_s3_location() region = get_region() -schema_name = "dbt_functional_test_snapshot01" +schema_name = "dbt_functional_test_01" def check_relation_rows(project, snapshot_name, count): @@ -83,4 +83,46 @@ class TestSnapshotTimestampGlue(BaseSnapshotTimestamp): def unique_schema(request, prefix) -> str: return schema_name + @pytest.fixture(scope='class', autouse=True) + def cleanup(self): + cleanup_s3_location(s3bucket + schema_name, region) + yield + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seeds": { + "+file_format": "delta", + "quote_columns": False, + }, + "snapshots": { + "+file_format": "delta", + "+updated_at": "current_timestamp()", + "quote_columns": False, + }, + "quoting": { + "database": False, + "schema": False, + "identifier": False + }, + } + + def test_snapshot_timestamp(self, project): + # seed command + results = run_dbt(["seed"]) + assert len(results) == 3 + + # snapshot command + results = run_dbt(["snapshot"]) + assert len(results) == 1 + + # snapshot has 10 rows + check_relation_rows(project, "ts_snapshot", 10) + + # point at the "added" seed so the snapshot sees 10 new rows + results = run_dbt(["snapshot", "--vars", "seed_name: added"]) + + # snapshot now has 20 rows + check_relation_rows(project, "ts_snapshot", 20) + pass From 5b13ab1fbe20e9ed9da9b149de68d2ca2f784b64 Mon Sep 17 00:00:00 2001 From: Noritaka Sekiyama Date: Tue, 14 Nov 2023 16:03:02 +0900 Subject: [PATCH 3/5] Added TestDocsGenReferencesGlue --- tests/functional/adapter/test_docs.py | 131 +++++++++++++++++++++++++- 1 file changed, 126 insertions(+), 5 deletions(-) diff --git a/tests/functional/adapter/test_docs.py b/tests/functional/adapter/test_docs.py index e8a86f20..aa1bd75c 100644 --- a/tests/functional/adapter/test_docs.py +++ b/tests/functional/adapter/test_docs.py @@ -1,10 +1,10 @@ +import os import pytest - -from dbt.tests.adapter.basic.test_docs_generate import (BaseDocsGenerate, - BaseDocsGenReferences) +from dbt.tests.util import get_artifact +from dbt.tests.adapter.basic.test_docs_generate import BaseDocsGenerate, BaseDocsGenReferences from dbt.tests.adapter.basic.expected_catalog import no_stats -schema_name = "dbt_functional_test_docs01" +schema_name = "dbt_functional_test_01" class TestDocsGenerate(BaseDocsGenerate): @@ -113,4 +113,125 @@ class TestDocsGenReferencesGlue(BaseDocsGenReferences): def unique_schema(request, prefix) -> str: return schema_name - pass + @pytest.fixture(scope="class") + def expected_catalog(self, project, profile_user): + role = None + id_type = "double" + text_type = "string" + time_type = "string" + view_type = "view" + table_type = "table" + model_stats = no_stats() + bigint_type = "bigint" + seed_stats = None + case = None + case_columns = False + view_summary_stats = None + + if case is None: + def case(x): + return x + + col_case = case if case_columns else lambda x: x + + if seed_stats is None: + seed_stats = model_stats + + if view_summary_stats is None: + view_summary_stats = model_stats + + my_schema_name = case(project.test_schema) + + summary_columns = { + "first_name": { + "name": "first_name", + "index": 0, + "type": text_type, + "comment": None, + }, + "ct": { + "name": "ct", + "index": 0, + "type": bigint_type, + "comment": None, + }, + } + + seed_columns = { + "id": { + "name": col_case("id"), + "index": 0, + "type": id_type, + "comment": None, + }, + "first_name": { + "name": col_case("first_name"), + "index": 0, + "type": text_type, + "comment": None, + }, + "email": { + "name": col_case("email"), + "index": 0, + "type": text_type, + "comment": None, + }, + "ip_address": { + "name": col_case("ip_address"), + "index": 0, + "type": text_type, + "comment": None, + }, + "updated_at": { + "name": col_case("updated_at"), + "index": 0, + "type": time_type, + "comment": None, + }, + } + return { + "nodes": { + "seed.test.seed": { + "unique_id": "seed.test.seed", + "metadata": { + "schema": my_schema_name, + "database": my_schema_name, + "name": case("seed"), + "type": table_type, + "comment": None, + "owner": role, + }, + "stats": seed_stats, + "columns": seed_columns, + }, + "model.test.ephemeral_summary": { + "unique_id": "model.test.ephemeral_summary", + "metadata": { + "schema": my_schema_name, + "database": my_schema_name, + "name": case("ephemeral_summary"), + "type": table_type, + "comment": None, + "owner": role, + }, + "stats": model_stats, + "columns": summary_columns, + }, + "model.test.view_summary": { + "unique_id": "model.test.view_summary", + "metadata": { + "schema": my_schema_name, + "database": my_schema_name, + "name": case("view_summary"), + "type": view_type, + "comment": None, + "owner": role, + }, + "stats": view_summary_stats, + "columns": summary_columns, + }, + }, + "sources": {} + } + + pass From ac2933087364524f5219cbd121b4a84012327f5b Mon Sep 17 00:00:00 2001 From: menuetb <83284881+menuetb@users.noreply.github.com> Date: Tue, 14 Nov 2023 09:14:30 +0100 Subject: [PATCH 4/5] fix TestTableMatGlue and TestValidateConnectionGlue --- tests/functional/adapter/test_basic.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/functional/adapter/test_basic.py b/tests/functional/adapter/test_basic.py index 3fc926c7..9938c152 100644 --- a/tests/functional/adapter/test_basic.py +++ b/tests/functional/adapter/test_basic.py @@ -48,6 +48,10 @@ class TestBaseCachingGlue(BaseAdapterMethod): + @pytest.fixture(scope="class") + def unique_schema(request, prefix) -> str: + return schema_name + pass @@ -265,8 +269,17 @@ def test_generic_tests(self, project): class TestTableMatGlue(BaseTableMaterialization): + @pytest.fixture(scope="class") + def unique_schema(request, prefix) -> str: + return schema_name + pass class TestValidateConnectionGlue(BaseValidateConnection): + @pytest.fixture(scope="class") + def unique_schema(request, prefix) -> str: + return schema_name + pass + From 6bce95ec589513663bcf4d86b6b347bc8c0c2a4b Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Tue, 14 Nov 2023 16:39:45 +0900 Subject: [PATCH 5/5] Skip TestBaseCachingGlue Also this fixes the execution error --- dbt/adapters/glue/connections.py | 2 +- tests/functional/adapter/test_basic.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/dbt/adapters/glue/connections.py b/dbt/adapters/glue/connections.py index 43cd4423..ad2006e8 100644 --- a/dbt/adapters/glue/connections.py +++ b/dbt/adapters/glue/connections.py @@ -101,7 +101,7 @@ def get_response(cls, cursor) -> AdapterResponse: def get_result_from_cursor(cls, cursor: GlueCursor, limit: Optional[int]) -> agate.Table: data: List[Any] = [] column_names: List[str] = [] - if cursor.description is not None: + if not cursor.description: column_names = [col[0] for col in cursor.description()] if limit: rows = cursor.fetchmany(limit) diff --git a/tests/functional/adapter/test_basic.py b/tests/functional/adapter/test_basic.py index 3fc926c7..84a12303 100644 --- a/tests/functional/adapter/test_basic.py +++ b/tests/functional/adapter/test_basic.py @@ -47,6 +47,10 @@ base_materialized_var_sql = config_materialized_var + config_incremental_strategy + model_base +@pytest.mark.skip( + reason="Fails because the test tries to fetch the table metadata during the compile step, " + "before the models are actually run. Not sure how this test is intended to work." +) class TestBaseCachingGlue(BaseAdapterMethod): pass