From 48a35568aab4534df54e628618022baf00a6750e Mon Sep 17 00:00:00 2001 From: Sam Debruyn Date: Thu, 18 May 2023 13:07:42 +0200 Subject: [PATCH 01/33] bump version --- dbt/adapters/sqlserver/__version__.py | 2 +- dev_requirements.txt | 2 +- setup.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dbt/adapters/sqlserver/__version__.py b/dbt/adapters/sqlserver/__version__.py index 6abaa204..e3a0f015 100644 --- a/dbt/adapters/sqlserver/__version__.py +++ b/dbt/adapters/sqlserver/__version__.py @@ -1 +1 @@ -version = "1.4.1" +version = "1.5.0" diff --git a/dev_requirements.txt b/dev_requirements.txt index 5ee38610..f3e8eef0 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -4,7 +4,7 @@ wheel==0.40.0 pre-commit==2.21.0;python_version<"3.8" pre-commit==3.3.2;python_version>="3.8" pytest-dotenv==0.5.2 -dbt-tests-adapter~=1.4.5 +dbt-tests-adapter~=1.5.1 flaky==3.7.0 pytest-xdist==3.3.0 -e . diff --git a/setup.py b/setup.py index 2d2fe6ba..ef69c11c 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ package_name = "dbt-sqlserver" authors_list = ["Mikael Ene", "Anders Swanson", "Sam Debruyn", "Cor Zuurmond"] -dbt_version = "1.4" +dbt_version = "1.5" description = """A Microsoft SQL Server adapter plugin for dbt""" this_directory = os.path.abspath(os.path.dirname(__file__)) @@ -66,7 +66,7 @@ def run(self): packages=find_namespace_packages(include=["dbt", "dbt.*"]), include_package_data=True, install_requires=[ - "dbt-core~=1.4.5", + "dbt-core~=1.5.1", "pyodbc~=4.0.35,!=4.0.36,!=4.0.37", "azure-identity>=1.12.0", ], From f4e8f5dc1c42fed3dfefff83ba597d7f32a65ddb Mon Sep 17 00:00:00 2001 From: Sam Debruyn Date: Thu, 18 May 2023 13:07:52 +0200 Subject: [PATCH 02/33] add new basic tests --- tests/functional/adapter/test_basic.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/functional/adapter/test_basic.py b/tests/functional/adapter/test_basic.py index a263d590..6e61ca79 100644 --- a/tests/functional/adapter/test_basic.py +++ b/tests/functional/adapter/test_basic.py @@ -13,6 +13,7 @@ from dbt.tests.adapter.basic.test_singular_tests_ephemeral import BaseSingularTestsEphemeral from dbt.tests.adapter.basic.test_snapshot_check_cols import BaseSnapshotCheckCols from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp +from dbt.tests.adapter.basic.test_table_materialization import BaseTableMaterialization from dbt.tests.adapter.basic.test_validate_connection import BaseValidateConnection @@ -69,3 +70,7 @@ class TestBaseCachingSQLServer(BaseAdapterMethod): class TestValidateConnectionSQLServer(BaseValidateConnection): pass + + +class TestTableMatSQLServer(BaseTableMaterialization): + pass From cbe301a68b434034c5941c449a78d5976df709ef Mon Sep 17 00:00:00 2001 From: Sam Debruyn Date: Thu, 18 May 2023 13:08:00 +0200 Subject: [PATCH 03/33] add new caching tests --- tests/functional/adapter/test_caching.py | 34 ++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 tests/functional/adapter/test_caching.py diff --git a/tests/functional/adapter/test_caching.py b/tests/functional/adapter/test_caching.py new file mode 100644 index 00000000..dddbc1c5 --- /dev/null +++ b/tests/functional/adapter/test_caching.py @@ -0,0 +1,34 @@ +import pytest +from dbt.tests.adapter.caching.test_caching import ( + BaseCachingLowercaseModel, + BaseCachingSelectedSchemaOnly, + BaseCachingTest, + BaseCachingUppercaseModel, + model_sql, +) + + +class TestNoPopulateCacheSQLServer(BaseCachingTest): + @pytest.fixture(scope="class") + def models(self): + return { + "model.sql": model_sql, + } + + def test_cache(self, project): + # --no-populate-cache still allows the cache to populate all relations + # under a schema, so the behavior here remains the same as other tests + run_args = ["--no-populate-cache", "run"] + self.run_and_inspect_cache(project, run_args) + + +class TestCachingLowerCaseModelSQLServer(BaseCachingLowercaseModel): + pass + + +class TestCachingUppercaseModelSQLServer(BaseCachingUppercaseModel): + pass + + +class TestCachingSelectedSchemaOnlySQLServer(BaseCachingSelectedSchemaOnly): + pass From ff147eb5693e3d1fda030093133f451d520f53fd Mon Sep 17 00:00:00 2001 From: Sam Debruyn Date: Thu, 18 May 2023 13:08:09 +0200 Subject: [PATCH 04/33] add new column type tests --- tests/functional/adapter/test_column_types.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 tests/functional/adapter/test_column_types.py diff --git a/tests/functional/adapter/test_column_types.py b/tests/functional/adapter/test_column_types.py new file mode 100644 index 00000000..6ba4f96a --- /dev/null +++ b/tests/functional/adapter/test_column_types.py @@ -0,0 +1,11 @@ +import pytest +from dbt.tests.adapter.column_types.test_column_types import BaseColumnTypes, model_sql, schema_yml + + +class TestColumnTypesSQLServer(BaseColumnTypes): + @pytest.fixture(scope="class") + def models(self): + return {"model.sql": model_sql, "schema.yml": schema_yml} + + def test_run_and_test(self, project): + self.run_and_test() From e56b15364d81e588c03fa2630242c213dda5d3dd Mon Sep 17 00:00:00 2001 From: Sam Debruyn Date: Thu, 18 May 2023 13:11:02 +0200 Subject: [PATCH 05/33] add new constraints tests --- tests/functional/adapter/test_constraints.py | 44 ++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 tests/functional/adapter/test_constraints.py diff --git a/tests/functional/adapter/test_constraints.py b/tests/functional/adapter/test_constraints.py new file mode 100644 index 00000000..1bebed91 --- /dev/null +++ b/tests/functional/adapter/test_constraints.py @@ -0,0 +1,44 @@ +from dbt.tests.adapter.constraints.test_constraints import ( + BaseConstraintsRollback, + BaseConstraintsRuntimeDdlEnforcement, + BaseIncrementalConstraintsColumnsEqual, + BaseIncrementalConstraintsRollback, + BaseIncrementalConstraintsRuntimeDdlEnforcement, + BaseModelConstraintsRuntimeEnforcement, + BaseTableConstraintsColumnsEqual, + BaseViewConstraintsColumnsEqual, +) + + +class TestModelConstraintsRuntimeEnforcementSQLServer(BaseModelConstraintsRuntimeEnforcement): + pass + + +class TestTableConstraintsColumnsEqualSQLServer(BaseTableConstraintsColumnsEqual): + pass + + +class TestViewConstraintsColumnsEqualSQLServer(BaseViewConstraintsColumnsEqual): + pass + + +class TestIncrementalConstraintsColumnsEqualSQLServer(BaseIncrementalConstraintsColumnsEqual): + pass + + +class TestTableConstraintsRuntimeDdlEnforcementSQLServer(BaseConstraintsRuntimeDdlEnforcement): + pass + + +class TestTableConstraintsRollbackSQLServer(BaseConstraintsRollback): + pass + + +class TestIncrementalConstraintsRuntimeDdlEnforcementSQLServer( + BaseIncrementalConstraintsRuntimeDdlEnforcement +): + pass + + +class TestIncrementalConstraintsRollbackSQLServer(BaseIncrementalConstraintsRollback): + pass From 581f72b1d02baa797edd865e2d463dc21f367d3e Mon Sep 17 00:00:00 2001 From: Sam Debruyn Date: Thu, 18 May 2023 13:18:15 +0200 Subject: [PATCH 06/33] simplify debug tests --- tests/functional/adapter/test_debug.py | 64 ++++---------------------- 1 file changed, 9 insertions(+), 55 deletions(-) diff --git a/tests/functional/adapter/test_debug.py b/tests/functional/adapter/test_debug.py index 84738fbc..a2160ce4 100644 --- a/tests/functional/adapter/test_debug.py +++ b/tests/functional/adapter/test_debug.py @@ -1,63 +1,17 @@ -import os -import re - -import yaml -from dbt.tests.adapter.dbt_debug.test_dbt_debug import BaseDebug, BaseDebugProfileVariable -from dbt.tests.util import run_dbt - - -class TestDebugSQLServer(BaseDebug): - def test_ok(self, project): - run_dbt(["debug"]) - assert "ERROR" not in self.capsys.readouterr().out - - def test_nopass(self, project): - run_dbt(["debug", "--target", "nopass"], expect_pass=False) - self.assertGotValue(re.compile(r"\s+profiles\.yml file"), "ERROR invalid") - - def test_wronguser(self, project): - run_dbt(["debug", "--target", "wronguser"], expect_pass=False) - self.assertGotValue(re.compile(r"\s+Connection test"), "ERROR") - - def test_empty_target(self, project): - run_dbt(["debug", "--target", "none_target"], expect_pass=False) - self.assertGotValue(re.compile(r"\s+output 'none_target'"), "misconfigured") +from dbt.tests.adapter.dbt_debug.test_dbt_debug import BaseDebugProfileVariable +from dbt.tests.adapter.dbt_debug.test_dbt_debug import ( + TestDebugInvalidProjectPostgres as BaseDebugInvalidProject, +) +from dbt.tests.adapter.dbt_debug.test_dbt_debug import TestDebugPostgres as BaseBaseDebug class TestDebugProfileVariableSQLServer(BaseDebugProfileVariable): pass -class TestDebugInvalidProjectSQLServer(BaseDebug): - def test_empty_project(self, project): - with open("dbt_project.yml", "w") as f: # noqa: F841 - pass - - run_dbt(["debug", "--profile", "test"], expect_pass=False) - splitout = self.capsys.readouterr().out.split("\n") - self.check_project(splitout) - - def test_badproject(self, project): - update_project = {"invalid-key": "not a valid key so this is bad project"} - - with open("dbt_project.yml", "w") as f: - yaml.safe_dump(update_project, f) - - run_dbt(["debug", "--profile", "test"], expect_pass=False) - splitout = self.capsys.readouterr().out.split("\n") - self.check_project(splitout) +class TestDebugInvalidProjectSQLServer(BaseDebugInvalidProject): + pass - def test_not_found_project(self, project): - run_dbt(["debug", "--project-dir", "nopass"], expect_pass=False) - splitout = self.capsys.readouterr().out.split("\n") - self.check_project(splitout, msg="ERROR not found") - def test_invalid_project_outside_current_dir(self, project): - # create a dbt_project.yml - project_config = {"invalid-key": "not a valid key in this project"} - os.makedirs("custom", exist_ok=True) - with open("custom/dbt_project.yml", "w") as f: - yaml.safe_dump(project_config, f, default_flow_style=True) - run_dbt(["debug", "--project-dir", "custom"], expect_pass=False) - splitout = self.capsys.readouterr().out.split("\n") - self.check_project(splitout) +class TestDebugSQLServer(BaseBaseDebug): + pass From 6573d0c969cb8cef6a7c704cb452627b02e5217a Mon Sep 17 00:00:00 2001 From: Sam Debruyn Date: Thu, 18 May 2023 13:18:25 +0200 Subject: [PATCH 07/33] rename data type tests --- .../adapter/{test_data_types.py => test_utils_data_types.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/functional/adapter/{test_data_types.py => test_utils_data_types.py} (100%) diff --git a/tests/functional/adapter/test_data_types.py b/tests/functional/adapter/test_utils_data_types.py similarity index 100% rename from tests/functional/adapter/test_data_types.py rename to tests/functional/adapter/test_utils_data_types.py From 9e5c8cdfcf54f6b3b832191f4969896ea9fd45f5 Mon Sep 17 00:00:00 2001 From: Sam Debruyn Date: Thu, 18 May 2023 13:20:48 +0200 Subject: [PATCH 08/33] move docs to basic --- tests/functional/adapter/test_basic.py | 90 +++++++++++++++++++++++++ tests/functional/adapter/test_docs.py | 91 -------------------------- 2 files changed, 90 insertions(+), 91 deletions(-) delete mode 100644 tests/functional/adapter/test_docs.py diff --git a/tests/functional/adapter/test_basic.py b/tests/functional/adapter/test_basic.py index 6e61ca79..7d743364 100644 --- a/tests/functional/adapter/test_basic.py +++ b/tests/functional/adapter/test_basic.py @@ -1,7 +1,22 @@ +import os + import pytest +from dbt.tests.adapter.basic.expected_catalog import ( + base_expected_catalog, + expected_references_catalog, + no_stats, +) from dbt.tests.adapter.basic.files import incremental_not_schema_change_sql from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod from dbt.tests.adapter.basic.test_base import BaseSimpleMaterializations +from dbt.tests.adapter.basic.test_docs_generate import ( + BaseDocsGenerate, + BaseDocsGenReferences, + ref_models__docs_md, + ref_models__ephemeral_copy_sql, + ref_models__schema_yml, + ref_sources__schema_yml, +) from dbt.tests.adapter.basic.test_empty import BaseEmpty from dbt.tests.adapter.basic.test_ephemeral import BaseEphemeral from dbt.tests.adapter.basic.test_generic_tests import BaseGenericTests @@ -74,3 +89,78 @@ class TestValidateConnectionSQLServer(BaseValidateConnection): class TestTableMatSQLServer(BaseTableMaterialization): pass + + +class TestDocsGenerateSQLServer(BaseDocsGenerate): + @staticmethod + @pytest.fixture(scope="class") + def dbt_profile_target_update(): + return {"schema_authorization": "{{ env_var('DBT_TEST_USER_1') }}"} + + @pytest.fixture(scope="class") + def expected_catalog(self, project): + return base_expected_catalog( + project, + role=os.getenv("DBT_TEST_USER_1"), + id_type="int", + text_type="varchar", + time_type="datetime", + view_type="VIEW", + table_type="BASE TABLE", + model_stats=no_stats(), + ) + + +class TestDocsGenReferencesSQLServer(BaseDocsGenReferences): + @staticmethod + @pytest.fixture(scope="class") + def dbt_profile_target_update(): + return {"schema_authorization": "{{ env_var('DBT_TEST_USER_1') }}"} + + @pytest.fixture(scope="class") + def expected_catalog(self, project): + return expected_references_catalog( + project, + role=os.getenv("DBT_TEST_USER_1"), + id_type="int", + text_type="varchar", + time_type="datetime", + bigint_type="int", + view_type="VIEW", + table_type="BASE TABLE", + model_stats=no_stats(), + ) + + @pytest.fixture(scope="class") + def models(self): + ref_models__ephemeral_summary_sql_no_order_by = """ + {{ + config( + materialized = "table" + ) + }} + + select first_name, count(*) as ct from {{ref('ephemeral_copy')}} + group by first_name + """ + + ref_models__view_summary_sql_no_order_by = """ + {{ + config( + materialized = "view" + ) + }} + + select first_name, ct from {{ref('ephemeral_summary')}} + """ + + return { + "schema.yml": ref_models__schema_yml, + "sources.yml": ref_sources__schema_yml, + # order by not allowed in VIEWS + "view_summary.sql": ref_models__view_summary_sql_no_order_by, + # order by not allowed in CTEs + "ephemeral_summary.sql": ref_models__ephemeral_summary_sql_no_order_by, + "ephemeral_copy.sql": ref_models__ephemeral_copy_sql, + "docs.md": ref_models__docs_md, + } diff --git a/tests/functional/adapter/test_docs.py b/tests/functional/adapter/test_docs.py deleted file mode 100644 index c9039950..00000000 --- a/tests/functional/adapter/test_docs.py +++ /dev/null @@ -1,91 +0,0 @@ -import os - -import pytest -from dbt.tests.adapter.basic.expected_catalog import ( - base_expected_catalog, - expected_references_catalog, - no_stats, -) -from dbt.tests.adapter.basic.test_docs_generate import ( - BaseDocsGenerate, - BaseDocsGenReferences, - ref_models__docs_md, - ref_models__ephemeral_copy_sql, - ref_models__schema_yml, - ref_sources__schema_yml, -) - - -class TestDocsGenerateSQLServer(BaseDocsGenerate): - @staticmethod - @pytest.fixture(scope="class") - def dbt_profile_target_update(): - return {"schema_authorization": "{{ env_var('DBT_TEST_USER_1') }}"} - - @pytest.fixture(scope="class") - def expected_catalog(self, project): - return base_expected_catalog( - project, - role=os.getenv("DBT_TEST_USER_1"), - id_type="int", - text_type="varchar", - time_type="datetime", - view_type="VIEW", - table_type="BASE TABLE", - model_stats=no_stats(), - ) - - -class TestDocsGenReferencesSQLServer(BaseDocsGenReferences): - @staticmethod - @pytest.fixture(scope="class") - def dbt_profile_target_update(): - return {"schema_authorization": "{{ env_var('DBT_TEST_USER_1') }}"} - - @pytest.fixture(scope="class") - def expected_catalog(self, project): - return expected_references_catalog( - project, - role=os.getenv("DBT_TEST_USER_1"), - id_type="int", - text_type="varchar", - time_type="datetime", - bigint_type="int", - view_type="VIEW", - table_type="BASE TABLE", - model_stats=no_stats(), - ) - - @pytest.fixture(scope="class") - def models(self): - ref_models__ephemeral_summary_sql_no_order_by = """ - {{ - config( - materialized = "table" - ) - }} - - select first_name, count(*) as ct from {{ref('ephemeral_copy')}} - group by first_name - """ - - ref_models__view_summary_sql_no_order_by = """ - {{ - config( - materialized = "view" - ) - }} - - select first_name, ct from {{ref('ephemeral_summary')}} - """ - - return { - "schema.yml": ref_models__schema_yml, - "sources.yml": ref_sources__schema_yml, - # order by not allowed in VIEWS - "view_summary.sql": ref_models__view_summary_sql_no_order_by, - # order by not allowed in CTEs - "ephemeral_summary.sql": ref_models__ephemeral_summary_sql_no_order_by, - "ephemeral_copy.sql": ref_models__ephemeral_copy_sql, - "docs.md": ref_models__docs_md, - } From bd56d0b33ea220f3b99ee2c1aa77092cd4a5f849 Mon Sep 17 00:00:00 2001 From: Sam Debruyn Date: Thu, 18 May 2023 13:35:52 +0200 Subject: [PATCH 09/33] add new ephemeral tests --- tests/functional/adapter/test_ephemeral.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/functional/adapter/test_ephemeral.py b/tests/functional/adapter/test_ephemeral.py index 2c11d6d6..66b1d60d 100644 --- a/tests/functional/adapter/test_ephemeral.py +++ b/tests/functional/adapter/test_ephemeral.py @@ -1,6 +1,10 @@ import pytest +from dbt.tests.adapter.ephemeral.test_ephemeral import BaseEphemeral +from dbt.tests.adapter.ephemeral.test_ephemeral import TestEphemeralMulti as BaseTestEphemeralMulti +from dbt.tests.adapter.ephemeral.test_ephemeral import ( + TestEphemeralNested as BaseTestEphemeralNested, +) from dbt.tests.adapter.ephemeral.test_ephemeral import ( - BaseEphemeral, ephemeral_errors__base__base_copy_sql, ephemeral_errors__base__base_sql, ephemeral_errors__dependent_sql, @@ -24,3 +28,13 @@ def test_ephemeral_error_handling(self, project): assert len(results) == 1 assert results[0].status == "skipped" assert "Compilation Error" in results[0].message + + +@pytest.mark.skip(reason="Ephemeral not supported") +class TestEphemeralNestedSQLServer(BaseTestEphemeralNested): + pass + + +@pytest.mark.skip(reason="Ephemeral not supported") +class TestEphemeralMultiSQLServer(BaseTestEphemeralMulti): + pass From 0363759008857958daeabc73d5b99b63d3442f49 Mon Sep 17 00:00:00 2001 From: Sam Debruyn Date: Thu, 18 May 2023 13:36:01 +0200 Subject: [PATCH 10/33] add new hooks tests --- tests/functional/adapter/test_hooks.py | 101 +++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 tests/functional/adapter/test_hooks.py diff --git a/tests/functional/adapter/test_hooks.py b/tests/functional/adapter/test_hooks.py new file mode 100644 index 00000000..5012ecda --- /dev/null +++ b/tests/functional/adapter/test_hooks.py @@ -0,0 +1,101 @@ +from dbt.tests.adapter.hooks.test_model_hooks import ( + TestDuplicateHooksInConfigs as BaseTestDuplicateHooksInConfigs, +) +from dbt.tests.adapter.hooks.test_model_hooks import TestHookRefs as BaseTestHookRefs +from dbt.tests.adapter.hooks.test_model_hooks import ( + TestHooksRefsOnSeeds as BaseTestHooksRefsOnSeeds, +) +from dbt.tests.adapter.hooks.test_model_hooks import ( + TestPrePostModelHooks as BaseTestPrePostModelHooks, +) +from dbt.tests.adapter.hooks.test_model_hooks import ( + TestPrePostModelHooksInConfig as BaseTestPrePostModelHooksInConfig, +) +from dbt.tests.adapter.hooks.test_model_hooks import ( + TestPrePostModelHooksInConfigKwargs as BaseTestPrePostModelHooksInConfigKwargs, +) +from dbt.tests.adapter.hooks.test_model_hooks import ( + TestPrePostModelHooksInConfigWithCount as BaseTestPrePostModelHooksInConfigWithCount, +) +from dbt.tests.adapter.hooks.test_model_hooks import ( + TestPrePostModelHooksOnSeeds as BaseTestPrePostModelHooksOnSeeds, +) +from dbt.tests.adapter.hooks.test_model_hooks import ( + TestPrePostModelHooksOnSeedsPlusPrefixed as BaseTestPrePostModelHooksOnSeedsPlusPrefixed, +) +from dbt.tests.adapter.hooks.test_model_hooks import ( + TestPrePostModelHooksOnSeedsPlusPrefixedWhitespace as BaseTPPMHOSPPrefixedWhitespace, +) +from dbt.tests.adapter.hooks.test_model_hooks import ( + TestPrePostModelHooksOnSnapshots as BaseTestPrePostModelHooksOnSnapshots, +) +from dbt.tests.adapter.hooks.test_model_hooks import ( + TestPrePostModelHooksUnderscores as BaseTestPrePostModelHooksUnderscores, +) +from dbt.tests.adapter.hooks.test_model_hooks import ( + TestPrePostSnapshotHooksInConfigKwargs as BaseTestPrePostSnapshotHooksInConfigKwargs, +) +from dbt.tests.adapter.hooks.test_run_hooks import TestAfterRunHooks as BaseTestAfterRunHooks +from dbt.tests.adapter.hooks.test_run_hooks import TestPrePostRunHooks as BaseTestPrePostRunHooks + + +class TestPrePostRunHooksSQLServer(BaseTestPrePostRunHooks): + pass + + +class TestAfterRunHooksSQLServer(BaseTestAfterRunHooks): + pass + + +class TestDuplicateHooksInConfigsSQLServer(BaseTestDuplicateHooksInConfigs): + pass + + +class TestHookRefsSQLServer(BaseTestHookRefs): + pass + + +class TestHooksRefsOnSeedsSQLServer(BaseTestHooksRefsOnSeeds): + pass + + +class TestPrePostModelHooksSQLServer(BaseTestPrePostModelHooks): + pass + + +class TestPrePostModelHooksInConfigSQLServer(BaseTestPrePostModelHooksInConfig): + pass + + +class TestPrePostModelHooksInConfigKwargsSQLServer(BaseTestPrePostModelHooksInConfigKwargs): + pass + + +class TestPrePostModelHooksInConfigWithCountSQLServer(BaseTestPrePostModelHooksInConfigWithCount): + pass + + +class TestPrePostModelHooksOnSeedsSQLServer(BaseTestPrePostModelHooksOnSeeds): + pass + + +class TestPrePostModelHooksOnSeedsPlusPrefixedSQLServer( + BaseTestPrePostModelHooksOnSeedsPlusPrefixed +): + pass + + +class TestPrePostModelHooksOnSeedsPlusPrefixedWhitespaceSQLServer(BaseTPPMHOSPPrefixedWhitespace): + pass + + +class TestPrePostModelHooksOnSnapshotsSQLServer(BaseTestPrePostModelHooksOnSnapshots): + pass + + +class TestPrePostModelHooksUnderscoresSQLServer(BaseTestPrePostModelHooksUnderscores): + pass + + +class TestPrePostSnapshotHooksInConfigKwargsSQLServer(BaseTestPrePostSnapshotHooksInConfigKwargs): + pass From d5f109cfbd03a24d3a2de88ca45d1106b76f78cd Mon Sep 17 00:00:00 2001 From: Sam Debruyn Date: Thu, 18 May 2023 15:49:23 +0200 Subject: [PATCH 11/33] wip --- tests/functional/adapter/test_simple_copy.py | 9 +++++++++ tests/functional/adapter/test_snapshot.py | 9 +++++++++ 2 files changed, 18 insertions(+) create mode 100644 tests/functional/adapter/test_simple_copy.py create mode 100644 tests/functional/adapter/test_snapshot.py diff --git a/tests/functional/adapter/test_simple_copy.py b/tests/functional/adapter/test_simple_copy.py new file mode 100644 index 00000000..60c502b3 --- /dev/null +++ b/tests/functional/adapter/test_simple_copy.py @@ -0,0 +1,9 @@ +from dbt.tests.adapter.simple_copy.test_simple_copy import EmptyModelsArentRunBase, SimpleCopyBase + + +class TestSimpleCopyBaseSQLServer(SimpleCopyBase): + pass + + +class TestEmptyModelsArentRunSQLServer(EmptyModelsArentRunBase): + pass diff --git a/tests/functional/adapter/test_snapshot.py b/tests/functional/adapter/test_snapshot.py new file mode 100644 index 00000000..5ef0c33d --- /dev/null +++ b/tests/functional/adapter/test_snapshot.py @@ -0,0 +1,9 @@ +from dbt.tests.adapter.simple_snapshot.test_snapshot import BaseSimpleSnapshot, BaseSnapshotCheck + + +class TestSnapshotSQLServer(BaseSimpleSnapshot): + pass + + +class TestSnapshotCheckSQLServer(BaseSnapshotCheck): + pass From 71739c22f74da55fe8ed71565a934beb5cee348c Mon Sep 17 00:00:00 2001 From: Sam Debruyn Date: Sun, 21 May 2023 20:16:37 +0200 Subject: [PATCH 12/33] use rc1 versions for now --- dev_requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index f3e8eef0..a8fa1a7a 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -4,7 +4,7 @@ wheel==0.40.0 pre-commit==2.21.0;python_version<"3.8" pre-commit==3.3.2;python_version>="3.8" pytest-dotenv==0.5.2 -dbt-tests-adapter~=1.5.1 +dbt-tests-adapter~=1.5.1rc1 flaky==3.7.0 pytest-xdist==3.3.0 -e . diff --git a/setup.py b/setup.py index ef69c11c..70bac492 100644 --- a/setup.py +++ b/setup.py @@ -66,7 +66,7 @@ def run(self): packages=find_namespace_packages(include=["dbt", "dbt.*"]), include_package_data=True, install_requires=[ - "dbt-core~=1.5.1", + "dbt-core~=1.5.1rc1", "pyodbc~=4.0.35,!=4.0.36,!=4.0.37", "azure-identity>=1.12.0", ], From 767b16fbd0518dc1cf75c88d11a3c45533f56ba5 Mon Sep 17 00:00:00 2001 From: Sam Debruyn Date: Sun, 21 May 2023 20:46:08 +0200 Subject: [PATCH 13/33] add new incremental tests --- tests/functional/adapter/test_incremental.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/functional/adapter/test_incremental.py b/tests/functional/adapter/test_incremental.py index c4d14c2d..cc4b619d 100644 --- a/tests/functional/adapter/test_incremental.py +++ b/tests/functional/adapter/test_incremental.py @@ -16,6 +16,10 @@ from dbt.tests.adapter.incremental.test_incremental_predicates import BaseIncrementalPredicates from dbt.tests.adapter.incremental.test_incremental_unique_id import BaseIncrementalUniqueKey +from tests.adapter.dbt.tests.adapter.incremental.test_incremental_merge_exclude_columns import ( + BaseMergeExcludeColumns, +) + _MODELS__INCREMENTAL_IGNORE = """ {{ config( @@ -118,3 +122,7 @@ class TestPredicatesDeleteInsertSQLServer(BaseIncrementalPredicates): @pytest.fixture(scope="class") def project_config_update(self): return {"models": {"+predicates": ["id != 2"], "+incremental_strategy": "delete+insert"}} + + +class TestBaseMergeExcludeColumnsSQLServer(BaseMergeExcludeColumns): + pass From 9e8c82cf45a6500aab9471028dd4bc6ca0f9a520 Mon Sep 17 00:00:00 2001 From: Sam Debruyn Date: Sun, 21 May 2023 20:48:01 +0200 Subject: [PATCH 14/33] add new persist docs tests --- tests/functional/adapter/test_persist_docs.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 tests/functional/adapter/test_persist_docs.py diff --git a/tests/functional/adapter/test_persist_docs.py b/tests/functional/adapter/test_persist_docs.py new file mode 100644 index 00000000..83850620 --- /dev/null +++ b/tests/functional/adapter/test_persist_docs.py @@ -0,0 +1,17 @@ +from tests.adapter.dbt.tests.adapter.persist_docs.test_persist_docs import ( + BasePersistDocs, + BasePersistDocsColumnMissing, + BasePersistDocsCommentOnQuotedColumn, +) + + +class TestPersistDocsSQLServer(BasePersistDocs): + pass + + +class TestPersistDocsColumnMissingSQLServer(BasePersistDocsColumnMissing): + pass + + +class TestPersistDocsCommentOnQuotedColumnSQLServer(BasePersistDocsCommentOnQuotedColumn): + pass From 53071bc44e432a2134542c21b91449d7da9aa20e Mon Sep 17 00:00:00 2001 From: Sam Debruyn Date: Sun, 21 May 2023 20:48:39 +0200 Subject: [PATCH 15/33] add new relations tests --- tests/functional/adapter/test_relations.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 tests/functional/adapter/test_relations.py diff --git a/tests/functional/adapter/test_relations.py b/tests/functional/adapter/test_relations.py new file mode 100644 index 00000000..0673e69b --- /dev/null +++ b/tests/functional/adapter/test_relations.py @@ -0,0 +1,7 @@ +from tests.adapter.dbt.tests.adapter.relations.test_changing_relation_type import ( + BaseChangeRelationTypeValidator, +) + + +class TestChangeRelationTypesSQLServer(BaseChangeRelationTypeValidator): + pass From 7ba8260344b87de0da21f441009b43a672d21136 Mon Sep 17 00:00:00 2001 From: Sam Debruyn Date: Sun, 21 May 2023 20:50:02 +0200 Subject: [PATCH 16/33] add new simple copy test --- tests/functional/adapter/test_simple_copy.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/functional/adapter/test_simple_copy.py b/tests/functional/adapter/test_simple_copy.py index 60c502b3..b3b04108 100644 --- a/tests/functional/adapter/test_simple_copy.py +++ b/tests/functional/adapter/test_simple_copy.py @@ -1,3 +1,6 @@ +from dbt.tests.adapter.simple_copy.test_copy_uppercase import ( + TestSimpleCopyUppercase as BaseSimpleCopyUppercase, +) from dbt.tests.adapter.simple_copy.test_simple_copy import EmptyModelsArentRunBase, SimpleCopyBase @@ -7,3 +10,7 @@ class TestSimpleCopyBaseSQLServer(SimpleCopyBase): class TestEmptyModelsArentRunSQLServer(EmptyModelsArentRunBase): pass + + +class TestSimpleCopyUppercaseSQLServer(BaseSimpleCopyUppercase): + pass From 74d91c02c2036cd3f02822f4c60d47194eb249a5 Mon Sep 17 00:00:00 2001 From: Sam Debruyn Date: Sun, 21 May 2023 20:57:55 +0200 Subject: [PATCH 17/33] split simple and non simple seed tests --- tests/functional/adapter/test_seed.py | 182 +------------------ tests/functional/adapter/test_simple_seed.py | 182 +++++++++++++++++++ 2 files changed, 183 insertions(+), 181 deletions(-) create mode 100644 tests/functional/adapter/test_simple_seed.py diff --git a/tests/functional/adapter/test_seed.py b/tests/functional/adapter/test_seed.py index 0eb26b66..eef49c42 100644 --- a/tests/functional/adapter/test_seed.py +++ b/tests/functional/adapter/test_seed.py @@ -1,188 +1,8 @@ import os import pytest -from dbt.tests.adapter.simple_seed.seeds import seeds__expected_sql from dbt.tests.adapter.simple_seed.test_seed import SeedConfigBase -from dbt.tests.adapter.simple_seed.test_seed import TestBasicSeedTests as BaseBasicSeedTests -from dbt.tests.adapter.simple_seed.test_seed import ( - TestSeedConfigFullRefreshOff as BaseSeedConfigFullRefreshOff, -) -from dbt.tests.adapter.simple_seed.test_seed import ( - TestSeedConfigFullRefreshOn as BaseSeedConfigFullRefreshOn, -) -from dbt.tests.adapter.simple_seed.test_seed import TestSeedCustomSchema as BaseSeedCustomSchema -from dbt.tests.adapter.simple_seed.test_seed import TestSeedParsing as BaseSeedParsing -from dbt.tests.adapter.simple_seed.test_seed import ( - TestSeedSpecificFormats as BaseSeedSpecificFormats, -) -from dbt.tests.adapter.simple_seed.test_seed import ( - TestSimpleSeedEnabledViaConfig as BaseSimpleSeedEnabledViaConfig, -) -from dbt.tests.adapter.simple_seed.test_seed_type_override import ( - BaseSimpleSeedColumnOverride, - seeds__disabled_in_config_csv, - seeds__enabled_in_config_csv, -) -from dbt.tests.util import get_connection, run_dbt - -from dbt.adapters.sqlserver import SQLServerAdapter - -fixed_setup_sql = seeds__expected_sql.replace("TIMESTAMP WITHOUT TIME ZONE", "DATETIME").replace( - "TEXT", "VARCHAR(255)" -) - -seeds__tricky_csv = """ -seed_id,seed_id_str,a_bool,looks_like_a_bool,a_date,looks_like_a_date,relative,weekday -1,1,1,1,2019-01-01 12:32:30,2019-01-01 12:32:30,tomorrow,Saturday -2,2,1,1,2019-01-01 12:32:31,2019-01-01 12:32:31,today,Sunday -3,3,1,1,2019-01-01 12:32:32,2019-01-01 12:32:32,yesterday,Monday -4,4,0,0,2019-01-01 01:32:32,2019-01-01 01:32:32,tomorrow,Saturday -5,5,0,0,2019-01-01 01:32:32,2019-01-01 01:32:32,today,Sunday -6,6,0,0,2019-01-01 01:32:32,2019-01-01 01:32:32,yesterday,Monday -""".lstrip() - -macros__schema_test = """ -{% test column_type(model, column_name, type) %} - - {% set cols = adapter.get_columns_in_relation(model) %} - - {% set col_types = {} %} - {% for col in cols %} - {% do col_types.update({col.name: col.data_type}) %} - {% endfor %} - - {% set col_type = col_types.get(column_name) %} - {% set col_type = 'text' if col_type and 'varchar' in col_type else col_type %} - - {% set validation_message = 'Got a column type of ' ~ col_type ~ ', expected ' ~ type %} - - {% set val = 0 if col_type == type else 1 %} - {% if val == 1 and execute %} - {{ log(validation_message, info=True) }} - {% endif %} - - select '{{ validation_message }}' as validation_error - from (select 1 as empty) as nothing - where {{ val }} = 1 - -{% endtest %} - -""" - -properties__schema_yml = """ -version: 2 -seeds: -- name: seed_enabled - columns: - - name: birthday - tests: - - column_type: - type: date - - name: seed_id - tests: - - column_type: - type: text - -- name: seed_tricky - columns: - - name: seed_id - tests: - - column_type: - type: int - - name: seed_id_str - tests: - - column_type: - type: text - - name: a_bool - tests: - - column_type: - type: int - - name: looks_like_a_bool - tests: - - column_type: - type: text - - name: a_date - tests: - - column_type: - type: datetime - - name: looks_like_a_date - tests: - - column_type: - type: text - - name: relative - tests: - - column_type: - type: text - - name: weekday - tests: - - column_type: - type: text -""" - - -class TestSimpleSeedColumnOverrideSQLServer(BaseSimpleSeedColumnOverride): - @pytest.fixture(scope="class") - def seeds(self): - return { - "seed_enabled.csv": seeds__enabled_in_config_csv, - "seed_disabled.csv": seeds__disabled_in_config_csv, - "seed_tricky.csv": seeds__tricky_csv, - } - - @pytest.fixture(scope="class") - def macros(self): - return {"schema_test.sql": macros__schema_test} - - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": properties__schema_yml, - } - - -class TestBasicSeedTestsSQLServer(BaseBasicSeedTests): - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project): - project.run_sql(fixed_setup_sql) - - -class TestSeedConfigFullRefreshOnSQLServer(BaseSeedConfigFullRefreshOn): - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project): - project.run_sql(fixed_setup_sql) - - -class TestSeedConfigFullRefreshOffSQLServer(BaseSeedConfigFullRefreshOff): - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project): - project.run_sql(fixed_setup_sql) - - -class TestSeedCustomSchemaSQLServer(BaseSeedCustomSchema): - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project): - project.run_sql(fixed_setup_sql) - - -class TestSimpleSeedEnabledViaConfigSQLServer(BaseSimpleSeedEnabledViaConfig): - @pytest.fixture(scope="function") - def clear_test_schema(self, project): - yield - adapter = project.adapter - assert isinstance(project.adapter, SQLServerAdapter) - with get_connection(project.adapter): - rel = adapter.Relation.create(database=project.database, schema=project.test_schema) - adapter.drop_schema(rel) - - -class TestSeedParsingSQLServer(BaseSeedParsing): - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project): - project.run_sql(fixed_setup_sql) - - -class TestSeedSpecificFormatsSQLServer(BaseSeedSpecificFormats): - pass +from dbt.tests.util import run_dbt class TestSeedBatchSizeMaxSQLServer(SeedConfigBase): diff --git a/tests/functional/adapter/test_simple_seed.py b/tests/functional/adapter/test_simple_seed.py new file mode 100644 index 00000000..8157566c --- /dev/null +++ b/tests/functional/adapter/test_simple_seed.py @@ -0,0 +1,182 @@ +import pytest +from dbt.tests.adapter.simple_seed.seeds import seeds__expected_sql +from dbt.tests.adapter.simple_seed.test_seed import TestBasicSeedTests as BaseBasicSeedTests +from dbt.tests.adapter.simple_seed.test_seed import ( + TestSeedConfigFullRefreshOff as BaseSeedConfigFullRefreshOff, +) +from dbt.tests.adapter.simple_seed.test_seed import ( + TestSeedConfigFullRefreshOn as BaseSeedConfigFullRefreshOn, +) +from dbt.tests.adapter.simple_seed.test_seed import TestSeedCustomSchema as BaseSeedCustomSchema +from dbt.tests.adapter.simple_seed.test_seed import TestSeedParsing as BaseSeedParsing +from dbt.tests.adapter.simple_seed.test_seed import ( + TestSeedSpecificFormats as BaseSeedSpecificFormats, +) +from dbt.tests.adapter.simple_seed.test_seed import ( + TestSimpleSeedEnabledViaConfig as BaseSimpleSeedEnabledViaConfig, +) +from dbt.tests.adapter.simple_seed.test_seed_type_override import ( + BaseSimpleSeedColumnOverride, + seeds__disabled_in_config_csv, + seeds__enabled_in_config_csv, +) +from dbt.tests.util import get_connection + +from dbt.adapters.sqlserver import SQLServerAdapter + +fixed_setup_sql = seeds__expected_sql.replace("TIMESTAMP WITHOUT TIME ZONE", "DATETIME").replace( + "TEXT", "VARCHAR(255)" +) + +seeds__tricky_csv = """ +seed_id,seed_id_str,a_bool,looks_like_a_bool,a_date,looks_like_a_date,relative,weekday +1,1,1,1,2019-01-01 12:32:30,2019-01-01 12:32:30,tomorrow,Saturday +2,2,1,1,2019-01-01 12:32:31,2019-01-01 12:32:31,today,Sunday +3,3,1,1,2019-01-01 12:32:32,2019-01-01 12:32:32,yesterday,Monday +4,4,0,0,2019-01-01 01:32:32,2019-01-01 01:32:32,tomorrow,Saturday +5,5,0,0,2019-01-01 01:32:32,2019-01-01 01:32:32,today,Sunday +6,6,0,0,2019-01-01 01:32:32,2019-01-01 01:32:32,yesterday,Monday +""".lstrip() + +macros__schema_test = """ +{% test column_type(model, column_name, type) %} + + {% set cols = adapter.get_columns_in_relation(model) %} + + {% set col_types = {} %} + {% for col in cols %} + {% do col_types.update({col.name: col.data_type}) %} + {% endfor %} + + {% set col_type = col_types.get(column_name) %} + {% set col_type = 'text' if col_type and 'varchar' in col_type else col_type %} + + {% set validation_message = 'Got a column type of ' ~ col_type ~ ', expected ' ~ type %} + + {% set val = 0 if col_type == type else 1 %} + {% if val == 1 and execute %} + {{ log(validation_message, info=True) }} + {% endif %} + + select '{{ validation_message }}' as validation_error + from (select 1 as empty) as nothing + where {{ val }} = 1 + +{% endtest %} + +""" + +properties__schema_yml = """ +version: 2 +seeds: +- name: seed_enabled + columns: + - name: birthday + tests: + - column_type: + type: date + - name: seed_id + tests: + - column_type: + type: text + +- name: seed_tricky + columns: + - name: seed_id + tests: + - column_type: + type: int + - name: seed_id_str + tests: + - column_type: + type: text + - name: a_bool + tests: + - column_type: + type: int + - name: looks_like_a_bool + tests: + - column_type: + type: text + - name: a_date + tests: + - column_type: + type: datetime + - name: looks_like_a_date + tests: + - column_type: + type: text + - name: relative + tests: + - column_type: + type: text + - name: weekday + tests: + - column_type: + type: text +""" + + +class TestSimpleSeedColumnOverrideSQLServer(BaseSimpleSeedColumnOverride): + @pytest.fixture(scope="class") + def seeds(self): + return { + "seed_enabled.csv": seeds__enabled_in_config_csv, + "seed_disabled.csv": seeds__disabled_in_config_csv, + "seed_tricky.csv": seeds__tricky_csv, + } + + @pytest.fixture(scope="class") + def macros(self): + return {"schema_test.sql": macros__schema_test} + + @pytest.fixture(scope="class") + def models(self): + return { + "schema.yml": properties__schema_yml, + } + + +class TestBasicSeedTestsSQLServer(BaseBasicSeedTests): + @pytest.fixture(scope="class", autouse=True) + def setUp(self, project): + project.run_sql(fixed_setup_sql) + + +class TestSeedConfigFullRefreshOnSQLServer(BaseSeedConfigFullRefreshOn): + @pytest.fixture(scope="class", autouse=True) + def setUp(self, project): + project.run_sql(fixed_setup_sql) + + +class TestSeedConfigFullRefreshOffSQLServer(BaseSeedConfigFullRefreshOff): + @pytest.fixture(scope="class", autouse=True) + def setUp(self, project): + project.run_sql(fixed_setup_sql) + + +class TestSeedCustomSchemaSQLServer(BaseSeedCustomSchema): + @pytest.fixture(scope="class", autouse=True) + def setUp(self, project): + project.run_sql(fixed_setup_sql) + + +class TestSimpleSeedEnabledViaConfigSQLServer(BaseSimpleSeedEnabledViaConfig): + @pytest.fixture(scope="function") + def clear_test_schema(self, project): + yield + adapter = project.adapter + assert isinstance(project.adapter, SQLServerAdapter) + with get_connection(project.adapter): + rel = adapter.Relation.create(database=project.database, schema=project.test_schema) + adapter.drop_schema(rel) + + +class TestSeedParsingSQLServer(BaseSeedParsing): + @pytest.fixture(scope="class", autouse=True) + def setUp(self, project): + project.run_sql(fixed_setup_sql) + + +class TestSeedSpecificFormatsSQLServer(BaseSeedSpecificFormats): + pass From 27166cfa4b2251a30c41fd12e7b11ae2a1298335 Mon Sep 17 00:00:00 2001 From: Sam Debruyn Date: Sun, 21 May 2023 21:00:33 +0200 Subject: [PATCH 18/33] rename simple snapshot tests --- .../adapter/{test_snapshot.py => test_simple_snapshot.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/functional/adapter/{test_snapshot.py => test_simple_snapshot.py} (100%) diff --git a/tests/functional/adapter/test_snapshot.py b/tests/functional/adapter/test_simple_snapshot.py similarity index 100% rename from tests/functional/adapter/test_snapshot.py rename to tests/functional/adapter/test_simple_snapshot.py From 6599edd59aa61289ea07eea6487335d1e87cc9fb Mon Sep 17 00:00:00 2001 From: Sam Debruyn Date: Sun, 21 May 2023 21:01:51 +0200 Subject: [PATCH 19/33] add store test failures --- tests/functional/adapter/test_store_test_failures.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 tests/functional/adapter/test_store_test_failures.py diff --git a/tests/functional/adapter/test_store_test_failures.py b/tests/functional/adapter/test_store_test_failures.py new file mode 100644 index 00000000..294ddcbf --- /dev/null +++ b/tests/functional/adapter/test_store_test_failures.py @@ -0,0 +1,7 @@ +from dbt.tests.adapter.store_test_failures_tests.test_store_test_failures import ( + TestStoreTestFailures as BaseStoreTestFailures, +) + + +class TestStoreTestFailuresSQLServer(BaseStoreTestFailures): + pass From e35933193f000b8d3c126b42cf9602cd2577bdf3 Mon Sep 17 00:00:00 2001 From: Sam Debruyn Date: Sun, 21 May 2023 21:12:30 +0200 Subject: [PATCH 20/33] update ref --- dbt/adapters/sqlserver/sql_server_adapter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt/adapters/sqlserver/sql_server_adapter.py b/dbt/adapters/sqlserver/sql_server_adapter.py index 6a621ac2..aa093d52 100644 --- a/dbt/adapters/sqlserver/sql_server_adapter.py +++ b/dbt/adapters/sqlserver/sql_server_adapter.py @@ -2,7 +2,7 @@ import agate from dbt.adapters.base.relation import BaseRelation -from dbt.adapters.cache import _make_ref_key_msg +from dbt.adapters.cache import _make_ref_key_dict from dbt.adapters.sql import SQLAdapter from dbt.adapters.sql.impl import CREATE_SCHEMA_MACRO_NAME from dbt.events.functions import fire_event @@ -20,7 +20,7 @@ class SQLServerAdapter(SQLAdapter): def create_schema(self, relation: BaseRelation) -> None: relation = relation.without_identifier() - fire_event(SchemaCreation(relation=_make_ref_key_msg(relation))) + fire_event(SchemaCreation(relation=_make_ref_key_dict(relation))) macro_name = CREATE_SCHEMA_MACRO_NAME kwargs = { "relation": relation, From ae0e94315af6565aa07ce01762e52111fa22e02b Mon Sep 17 00:00:00 2001 From: Sam Debruyn Date: Sun, 21 May 2023 21:20:41 +0200 Subject: [PATCH 21/33] remove hooks tests --- tests/functional/adapter/test_hooks.py | 101 ------------------------- 1 file changed, 101 deletions(-) delete mode 100644 tests/functional/adapter/test_hooks.py diff --git a/tests/functional/adapter/test_hooks.py b/tests/functional/adapter/test_hooks.py deleted file mode 100644 index 5012ecda..00000000 --- a/tests/functional/adapter/test_hooks.py +++ /dev/null @@ -1,101 +0,0 @@ -from dbt.tests.adapter.hooks.test_model_hooks import ( - TestDuplicateHooksInConfigs as BaseTestDuplicateHooksInConfigs, -) -from dbt.tests.adapter.hooks.test_model_hooks import TestHookRefs as BaseTestHookRefs -from dbt.tests.adapter.hooks.test_model_hooks import ( - TestHooksRefsOnSeeds as BaseTestHooksRefsOnSeeds, -) -from dbt.tests.adapter.hooks.test_model_hooks import ( - TestPrePostModelHooks as BaseTestPrePostModelHooks, -) -from dbt.tests.adapter.hooks.test_model_hooks import ( - TestPrePostModelHooksInConfig as BaseTestPrePostModelHooksInConfig, -) -from dbt.tests.adapter.hooks.test_model_hooks import ( - TestPrePostModelHooksInConfigKwargs as BaseTestPrePostModelHooksInConfigKwargs, -) -from dbt.tests.adapter.hooks.test_model_hooks import ( - TestPrePostModelHooksInConfigWithCount as BaseTestPrePostModelHooksInConfigWithCount, -) -from dbt.tests.adapter.hooks.test_model_hooks import ( - TestPrePostModelHooksOnSeeds as BaseTestPrePostModelHooksOnSeeds, -) -from dbt.tests.adapter.hooks.test_model_hooks import ( - TestPrePostModelHooksOnSeedsPlusPrefixed as BaseTestPrePostModelHooksOnSeedsPlusPrefixed, -) -from dbt.tests.adapter.hooks.test_model_hooks import ( - TestPrePostModelHooksOnSeedsPlusPrefixedWhitespace as BaseTPPMHOSPPrefixedWhitespace, -) -from dbt.tests.adapter.hooks.test_model_hooks import ( - TestPrePostModelHooksOnSnapshots as BaseTestPrePostModelHooksOnSnapshots, -) -from dbt.tests.adapter.hooks.test_model_hooks import ( - TestPrePostModelHooksUnderscores as BaseTestPrePostModelHooksUnderscores, -) -from dbt.tests.adapter.hooks.test_model_hooks import ( - TestPrePostSnapshotHooksInConfigKwargs as BaseTestPrePostSnapshotHooksInConfigKwargs, -) -from dbt.tests.adapter.hooks.test_run_hooks import TestAfterRunHooks as BaseTestAfterRunHooks -from dbt.tests.adapter.hooks.test_run_hooks import TestPrePostRunHooks as BaseTestPrePostRunHooks - - -class TestPrePostRunHooksSQLServer(BaseTestPrePostRunHooks): - pass - - -class TestAfterRunHooksSQLServer(BaseTestAfterRunHooks): - pass - - -class TestDuplicateHooksInConfigsSQLServer(BaseTestDuplicateHooksInConfigs): - pass - - -class TestHookRefsSQLServer(BaseTestHookRefs): - pass - - -class TestHooksRefsOnSeedsSQLServer(BaseTestHooksRefsOnSeeds): - pass - - -class TestPrePostModelHooksSQLServer(BaseTestPrePostModelHooks): - pass - - -class TestPrePostModelHooksInConfigSQLServer(BaseTestPrePostModelHooksInConfig): - pass - - -class TestPrePostModelHooksInConfigKwargsSQLServer(BaseTestPrePostModelHooksInConfigKwargs): - pass - - -class TestPrePostModelHooksInConfigWithCountSQLServer(BaseTestPrePostModelHooksInConfigWithCount): - pass - - -class TestPrePostModelHooksOnSeedsSQLServer(BaseTestPrePostModelHooksOnSeeds): - pass - - -class TestPrePostModelHooksOnSeedsPlusPrefixedSQLServer( - BaseTestPrePostModelHooksOnSeedsPlusPrefixed -): - pass - - -class TestPrePostModelHooksOnSeedsPlusPrefixedWhitespaceSQLServer(BaseTPPMHOSPPrefixedWhitespace): - pass - - -class TestPrePostModelHooksOnSnapshotsSQLServer(BaseTestPrePostModelHooksOnSnapshots): - pass - - -class TestPrePostModelHooksUnderscoresSQLServer(BaseTestPrePostModelHooksUnderscores): - pass - - -class TestPrePostSnapshotHooksInConfigKwargsSQLServer(BaseTestPrePostSnapshotHooksInConfigKwargs): - pass From a7bf3706dda263bffdc05b9ecdb3060f0cfa36dc Mon Sep 17 00:00:00 2001 From: Ty Schlichenmeyer Date: Thu, 19 Oct 2023 13:19:27 -0500 Subject: [PATCH 22/33] build(dev): give Dockerfile temporary root permissions for filesystem operations --- CONTRIBUTING.md | 21 ++++++++++++--------- devops/server.Dockerfile | 4 ++++ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f01e4bc0..4154aa19 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,8 +1,12 @@ # Development of the adapter -Python 3.10 is used for developing the adapter. To get started, bootstrap your environment as follows: +## Dependencies -Create a virtual environment, [pyenv](https://github.com/pyenv/pyenv) is used in the example: +A recent version of [Docker](https://docs.docker.com/get-docker/). + +Python 3.10 + +Create a virtual environment to properly isolate your Python dependencies. Here is an example of how to do so using the [pyenv](https://github.com/pyenv/pyenv) CLI: ```shell pyenv install 3.10.7 @@ -22,22 +26,21 @@ After running `make dev`, pre-commit will automatically validate your commits an ## Testing -The functional tests require a running SQL Server instance. You can easily spin up a local instance with the following command: +The functional tests require a running SQL Server instance. We'll spin one up using Docker Compose. First, copy the example `.env.test.sample` to create a new file `.env.test`, which contains the environment variables that will be read by Docker Compose: ```shell -make server +cp test.env.sample test.env ``` -This will use Docker Compose to spin up a local instance of SQL Server. Docker Compose is now bundled with Docker, so make sure to [install the latest version of Docker](https://docs.docker.com/get-docker/). +If desired, you can tweak the contents of `test.env` to test against a different database. -Next, tell our tests how they should connect to the local instance by creating a file called `test.env` in the root of the project. -You can use the provided `test.env.sample` as a base and if you started the server with `make server`, then this matches the instance running on your local machine. +Now, you can easily spin up a local SQL Server instance: ```shell -cp test.env.sample test.env +make server ``` -You can tweak the contents of this file to test against a different database. +This will use Docker Compose to fetch the image and run the container. Note that we need 3 users to be able to run tests related to the grants. The 3 users are defined by the following environment variables containing their usernames. diff --git a/devops/server.Dockerfile b/devops/server.Dockerfile index 6ab402dd..58ec1f6c 100644 --- a/devops/server.Dockerfile +++ b/devops/server.Dockerfile @@ -3,8 +3,12 @@ FROM mcr.microsoft.com/mssql/server:${MSSQL_VERSION}-latest ENV COLLATION="SQL_Latin1_General_CP1_CI_AS" +USER root + RUN mkdir -p /opt/init_scripts WORKDIR /opt/init_scripts COPY scripts/* /opt/init_scripts/ +USER mssql + ENTRYPOINT /bin/bash ./entrypoint.sh From 53d34bdb334172c06685477f05f6a146c05d9914 Mon Sep 17 00:00:00 2001 From: Ty Schlichenmeyer Date: Thu, 19 Oct 2023 14:13:20 -0500 Subject: [PATCH 23/33] build: match pre-commit python version to project --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e23b4d9f..ed57288f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ default_language_version: - python: python3.9 + python: python3.10 repos: - repo: 'https://github.com/pre-commit/pre-commit-hooks' rev: v4.4.0 From 8f93a7193e87ae00eeace66c8644a6933e74fccb Mon Sep 17 00:00:00 2001 From: Ty Schlichenmeyer Date: Thu, 19 Oct 2023 15:02:41 -0500 Subject: [PATCH 24/33] ci: install pyodbc dependencies in test container --- devops/server.Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/devops/server.Dockerfile b/devops/server.Dockerfile index 58ec1f6c..81077c6d 100644 --- a/devops/server.Dockerfile +++ b/devops/server.Dockerfile @@ -1,9 +1,11 @@ ARG MSSQL_VERSION="2022" FROM mcr.microsoft.com/mssql/server:${MSSQL_VERSION}-latest +USER root + ENV COLLATION="SQL_Latin1_General_CP1_CI_AS" -USER root +RUN apt update && apt install -y unixodbc RUN mkdir -p /opt/init_scripts WORKDIR /opt/init_scripts From e53572c44cc077ea277c4149a5a28581bea71056 Mon Sep 17 00:00:00 2001 From: Ty Schlichenmeyer Date: Tue, 31 Oct 2023 16:40:34 +0000 Subject: [PATCH 25/33] ci: devcontainer setup/use ruff for format/sorting imports --- .devcontainer/devcontainer.json | 19 ++++++++++++ .pre-commit-config.yaml | 54 +++++++++------------------------ Makefile | 4 +-- 3 files changed, 36 insertions(+), 41 deletions(-) create mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..0e31f685 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,19 @@ +{ + "name": "dbt-sqlserver", + "image": "mcr.microsoft.com/devcontainers/python:3.11", + "features": { + "ghcr.io/devcontainers/features/docker-in-docker:2": {}, + "ghcr.io/jlaundry/devcontainer-features/mssql-odbc-driver:1": {}, + "ghcr.io/devcontainers/features/azure-cli:1": {} + }, + "postCreateCommand": "cp test.env.sample test.env && make dev", + "customizations": { + "vscode": { + "extensions": [ + "ms-python.vscode-pylance", + "ms-vscode.makefile-tools", + "innoverio.vscode-dbt-power-user" + ] + } + } +} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ed57288f..60c90944 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,6 @@ default_language_version: - python: python3.10 + python: python3.11 + repos: - repo: 'https://github.com/pre-commit/pre-commit-hooks' rev: v4.4.0 @@ -20,6 +21,7 @@ repos: - id: fix-byte-order-marker - id: mixed-line-ending - id: check-docstring-first + - repo: 'https://github.com/adrienverge/yamllint' rev: v1.32.0 hooks: @@ -27,56 +29,29 @@ repos: args: - '-d {extends: default, rules: {line-length: disable, document-start: disable}}' - '-s' + - repo: 'https://github.com/MarcoGorelli/absolufy-imports' rev: v0.3.1 hooks: - id: absolufy-imports + - repo: 'https://github.com/hadialqattan/pycln' rev: v2.1.3 hooks: - id: pycln args: - '--all' - - repo: 'https://github.com/pycqa/isort' - rev: 5.12.0 + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.3 hooks: - - id: isort - args: - - '--profile' - - black - - '--atomic' - - '--line-length' - - '99' - - '--python-version' - - '39' - - repo: 'https://github.com/psf/black' - rev: 23.3.0 - hooks: - - id: black - args: - - '--line-length=99' - - '--target-version=py39' - - id: black - alias: black-check - stages: - - manual - args: - - '--line-length=99' - - '--target-version=py39' - - '--check' - - '--diff' - - repo: 'https://github.com/pycqa/flake8' - rev: 6.0.0 + - id: ruff + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.3 hooks: - - id: flake8 - args: - - '--max-line-length=99' - - id: flake8 - args: - - '--max-line-length=99' - alias: flake8-check - stages: - - manual + - id: ruff-format + - repo: 'https://github.com/pre-commit/mirrors-mypy' rev: v1.3.0 hooks: @@ -86,6 +61,7 @@ repos: - '--ignore-missing-imports' - '--explicit-package-bases' files: '^dbt/adapters' + - id: mypy alias: mypy-check stages: diff --git a/Makefile b/Makefile index 91863e5a..c98cc416 100644 --- a/Makefile +++ b/Makefile @@ -50,8 +50,8 @@ functional: ## Runs functional tests. test: ## Runs unit tests and code checks against staged changes. @\ pytest -n auto -ra -v tests/unit; \ - pre-commit run black-check --hook-stage manual | grep -v "INFO"; \ - pre-commit run flake8-check --hook-stage manual | grep -v "INFO"; \ + pre-commit run ruff-format --all-files --hook-stage manual | grep -v "INFO"; \ + pre-commit run ruff --all-files --hook-stage manual | grep -v "INFO"; \ pre-commit run mypy-check --hook-stage manual | grep -v "INFO" .PHONY: server From f4d7870859244212b597107f7aca18bd15b65491 Mon Sep 17 00:00:00 2001 From: Ty Schlichenmeyer Date: Tue, 31 Oct 2023 16:40:51 +0000 Subject: [PATCH 26/33] refactor: run ruff --- dbt/adapters/sqlserver/__init__.py | 4 +++- dbt/adapters/sqlserver/sql_server_adapter.py | 12 +++++++++--- .../sqlserver/sql_server_connection_manager.py | 12 +++++++++--- setup.py | 4 +++- tests/conftest.py | 5 ++++- tests/functional/adapter/test_basic.py | 4 +++- .../adapter/test_changing_relation_type.py | 4 +++- tests/functional/adapter/test_column_types.py | 6 +++++- tests/functional/adapter/test_concurrency.py | 5 ++++- tests/functional/adapter/test_constraints.py | 12 +++++++++--- tests/functional/adapter/test_debug.py | 4 +++- tests/functional/adapter/test_ephemeral.py | 4 +++- tests/functional/adapter/test_incremental.py | 15 ++++++++++++--- tests/functional/adapter/test_persist_docs.py | 4 +++- tests/functional/adapter/test_simple_copy.py | 5 ++++- tests/functional/adapter/test_simple_seed.py | 18 ++++++++++++------ .../functional/adapter/test_simple_snapshot.py | 5 ++++- tests/functional/adapter/test_sources.py | 5 ++++- tests/functional/adapter/test_utils.py | 8 ++++++-- .../adapter/test_utils_data_types.py | 4 +++- .../test_sql_server_connection_manager.py | 3 ++- 21 files changed, 108 insertions(+), 35 deletions(-) diff --git a/dbt/adapters/sqlserver/__init__.py b/dbt/adapters/sqlserver/__init__.py index eb6b2573..9ab0c8d1 100644 --- a/dbt/adapters/sqlserver/__init__.py +++ b/dbt/adapters/sqlserver/__init__.py @@ -3,7 +3,9 @@ from dbt.adapters.sqlserver.sql_server_adapter import SQLServerAdapter from dbt.adapters.sqlserver.sql_server_column import SQLServerColumn from dbt.adapters.sqlserver.sql_server_configs import SQLServerConfigs -from dbt.adapters.sqlserver.sql_server_connection_manager import SQLServerConnectionManager +from dbt.adapters.sqlserver.sql_server_connection_manager import ( + SQLServerConnectionManager, +) from dbt.adapters.sqlserver.sql_server_credentials import SQLServerCredentials from dbt.include import sqlserver diff --git a/dbt/adapters/sqlserver/sql_server_adapter.py b/dbt/adapters/sqlserver/sql_server_adapter.py index aa093d52..33456779 100644 --- a/dbt/adapters/sqlserver/sql_server_adapter.py +++ b/dbt/adapters/sqlserver/sql_server_adapter.py @@ -10,7 +10,9 @@ from dbt.adapters.sqlserver.sql_server_column import SQLServerColumn from dbt.adapters.sqlserver.sql_server_configs import SQLServerConfigs -from dbt.adapters.sqlserver.sql_server_connection_manager import SQLServerConnectionManager +from dbt.adapters.sqlserver.sql_server_connection_manager import ( + SQLServerConnectionManager, +) class SQLServerAdapter(SQLAdapter): @@ -27,7 +29,9 @@ def create_schema(self, relation: BaseRelation) -> None: } if self.config.credentials.schema_authorization: - kwargs["schema_authorization"] = self.config.credentials.schema_authorization + kwargs[ + "schema_authorization" + ] = self.config.credentials.schema_authorization macro_name = "sqlserver__create_schema_with_authorization" self.execute_macro(macro_name, kwargs=kwargs) @@ -64,7 +68,9 @@ def convert_time_type(cls, agate_table, col_idx): return "datetime" # Methods used in adapter tests - def timestamp_add_sql(self, add_to: str, number: int = 1, interval: str = "hour") -> str: + def timestamp_add_sql( + self, add_to: str, number: int = 1, interval: str = "hour" + ) -> str: # note: 'interval' is not supported for T-SQL # for backwards compatibility, we're compelled to set some sort of # default. A lot of searching has lead me to believe that the diff --git a/dbt/adapters/sqlserver/sql_server_connection_manager.py b/dbt/adapters/sqlserver/sql_server_connection_manager.py index f5ac8546..17b97b22 100644 --- a/dbt/adapters/sqlserver/sql_server_connection_manager.py +++ b/dbt/adapters/sqlserver/sql_server_connection_manager.py @@ -161,7 +161,9 @@ def get_sp_access_token(credentials: SQLServerCredentials) -> AccessToken: The access token. """ token = ClientSecretCredential( - str(credentials.tenant_id), str(credentials.client_id), str(credentials.client_secret) + str(credentials.tenant_id), + str(credentials.client_id), + str(credentials.client_secret), ).get_token(AZURE_CREDENTIAL_SCOPE) return token @@ -200,7 +202,9 @@ def get_pyodbc_attrs_before(credentials: SQLServerCredentials) -> Dict: authentication = str(credentials.authentication).lower() if authentication in AZURE_AUTH_FUNCTIONS: - time_remaining = (_TOKEN.expires_on - time.time()) if _TOKEN else MAX_REMAINING_TIME + time_remaining = ( + (_TOKEN.expires_on - time.time()) if _TOKEN else MAX_REMAINING_TIME + ) if _TOKEN is None or (time_remaining < MAX_REMAINING_TIME): azure_auth_function = AZURE_AUTH_FUNCTIONS[authentication] @@ -341,7 +345,9 @@ def open(cls, connection: Connection) -> Connection: con_str.append(bool_to_connection_string_arg("encrypt", credentials.encrypt)) con_str.append( - bool_to_connection_string_arg("TrustServerCertificate", credentials.trust_cert) + bool_to_connection_string_arg( + "TrustServerCertificate", credentials.trust_cert + ) ) plugin_version = __version__.version diff --git a/setup.py b/setup.py index a2823276..d2c34eb9 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,9 @@ # get this from a separate file def _dbt_sqlserver_version(): - _version_path = os.path.join(this_directory, "dbt", "adapters", "sqlserver", "__version__.py") + _version_path = os.path.join( + this_directory, "dbt", "adapters", "sqlserver", "__version__.py" + ) _version_pattern = r"""version\s*=\s*["'](.+)["']""" with open(_version_path) as f: match = re.search(_version_pattern, f.read().strip()) diff --git a/tests/conftest.py b/tests/conftest.py index 540ee302..f71ea16e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -161,5 +161,8 @@ def skip_by_profile_type(request: FixtureRequest): pytest.skip(f"Skipped on '{profile_type}' profile") if request.node.get_closest_marker("only_with_profile"): - if profile_type not in request.node.get_closest_marker("only_with_profile").args: + if ( + profile_type + not in request.node.get_closest_marker("only_with_profile").args + ): pytest.skip(f"Skipped on '{profile_type}' profile") diff --git a/tests/functional/adapter/test_basic.py b/tests/functional/adapter/test_basic.py index 7d743364..0efcc5a3 100644 --- a/tests/functional/adapter/test_basic.py +++ b/tests/functional/adapter/test_basic.py @@ -25,7 +25,9 @@ BaseIncrementalNotSchemaChange, ) from dbt.tests.adapter.basic.test_singular_tests import BaseSingularTests -from dbt.tests.adapter.basic.test_singular_tests_ephemeral import BaseSingularTestsEphemeral +from dbt.tests.adapter.basic.test_singular_tests_ephemeral import ( + BaseSingularTestsEphemeral, +) from dbt.tests.adapter.basic.test_snapshot_check_cols import BaseSnapshotCheckCols from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp from dbt.tests.adapter.basic.test_table_materialization import BaseTableMaterialization diff --git a/tests/functional/adapter/test_changing_relation_type.py b/tests/functional/adapter/test_changing_relation_type.py index aaa43fa0..e273ae01 100644 --- a/tests/functional/adapter/test_changing_relation_type.py +++ b/tests/functional/adapter/test_changing_relation_type.py @@ -1,4 +1,6 @@ -from dbt.tests.adapter.relations.test_changing_relation_type import BaseChangeRelationTypeValidator +from dbt.tests.adapter.relations.test_changing_relation_type import ( + BaseChangeRelationTypeValidator, +) class TestChangeRelationTypesSQLServer(BaseChangeRelationTypeValidator): diff --git a/tests/functional/adapter/test_column_types.py b/tests/functional/adapter/test_column_types.py index 6ba4f96a..2afa2855 100644 --- a/tests/functional/adapter/test_column_types.py +++ b/tests/functional/adapter/test_column_types.py @@ -1,5 +1,9 @@ import pytest -from dbt.tests.adapter.column_types.test_column_types import BaseColumnTypes, model_sql, schema_yml +from dbt.tests.adapter.column_types.test_column_types import ( + BaseColumnTypes, + model_sql, + schema_yml, +) class TestColumnTypesSQLServer(BaseColumnTypes): diff --git a/tests/functional/adapter/test_concurrency.py b/tests/functional/adapter/test_concurrency.py index f846f07a..ae0d4a50 100644 --- a/tests/functional/adapter/test_concurrency.py +++ b/tests/functional/adapter/test_concurrency.py @@ -1,4 +1,7 @@ -from dbt.tests.adapter.concurrency.test_concurrency import BaseConcurrency, seeds__update_csv +from dbt.tests.adapter.concurrency.test_concurrency import ( + BaseConcurrency, + seeds__update_csv, +) from dbt.tests.util import ( check_relations_equal, check_table_does_not_exist, diff --git a/tests/functional/adapter/test_constraints.py b/tests/functional/adapter/test_constraints.py index 1bebed91..ee03bbbb 100644 --- a/tests/functional/adapter/test_constraints.py +++ b/tests/functional/adapter/test_constraints.py @@ -10,7 +10,9 @@ ) -class TestModelConstraintsRuntimeEnforcementSQLServer(BaseModelConstraintsRuntimeEnforcement): +class TestModelConstraintsRuntimeEnforcementSQLServer( + BaseModelConstraintsRuntimeEnforcement +): pass @@ -22,11 +24,15 @@ class TestViewConstraintsColumnsEqualSQLServer(BaseViewConstraintsColumnsEqual): pass -class TestIncrementalConstraintsColumnsEqualSQLServer(BaseIncrementalConstraintsColumnsEqual): +class TestIncrementalConstraintsColumnsEqualSQLServer( + BaseIncrementalConstraintsColumnsEqual +): pass -class TestTableConstraintsRuntimeDdlEnforcementSQLServer(BaseConstraintsRuntimeDdlEnforcement): +class TestTableConstraintsRuntimeDdlEnforcementSQLServer( + BaseConstraintsRuntimeDdlEnforcement +): pass diff --git a/tests/functional/adapter/test_debug.py b/tests/functional/adapter/test_debug.py index a2160ce4..c86cb62d 100644 --- a/tests/functional/adapter/test_debug.py +++ b/tests/functional/adapter/test_debug.py @@ -2,7 +2,9 @@ from dbt.tests.adapter.dbt_debug.test_dbt_debug import ( TestDebugInvalidProjectPostgres as BaseDebugInvalidProject, ) -from dbt.tests.adapter.dbt_debug.test_dbt_debug import TestDebugPostgres as BaseBaseDebug +from dbt.tests.adapter.dbt_debug.test_dbt_debug import ( + TestDebugPostgres as BaseBaseDebug, +) class TestDebugProfileVariableSQLServer(BaseDebugProfileVariable): diff --git a/tests/functional/adapter/test_ephemeral.py b/tests/functional/adapter/test_ephemeral.py index 66b1d60d..8163d8f1 100644 --- a/tests/functional/adapter/test_ephemeral.py +++ b/tests/functional/adapter/test_ephemeral.py @@ -1,6 +1,8 @@ import pytest from dbt.tests.adapter.ephemeral.test_ephemeral import BaseEphemeral -from dbt.tests.adapter.ephemeral.test_ephemeral import TestEphemeralMulti as BaseTestEphemeralMulti +from dbt.tests.adapter.ephemeral.test_ephemeral import ( + TestEphemeralMulti as BaseTestEphemeralMulti, +) from dbt.tests.adapter.ephemeral.test_ephemeral import ( TestEphemeralNested as BaseTestEphemeralNested, ) diff --git a/tests/functional/adapter/test_incremental.py b/tests/functional/adapter/test_incremental.py index cc4b619d..132128ba 100644 --- a/tests/functional/adapter/test_incremental.py +++ b/tests/functional/adapter/test_incremental.py @@ -13,8 +13,12 @@ from dbt.tests.adapter.incremental.test_incremental_on_schema_change import ( BaseIncrementalOnSchemaChange, ) -from dbt.tests.adapter.incremental.test_incremental_predicates import BaseIncrementalPredicates -from dbt.tests.adapter.incremental.test_incremental_unique_id import BaseIncrementalUniqueKey +from dbt.tests.adapter.incremental.test_incremental_predicates import ( + BaseIncrementalPredicates, +) +from dbt.tests.adapter.incremental.test_incremental_unique_id import ( + BaseIncrementalUniqueKey, +) from tests.adapter.dbt.tests.adapter.incremental.test_incremental_merge_exclude_columns import ( BaseMergeExcludeColumns, @@ -121,7 +125,12 @@ class TestIncrementalPredicatesDeleteInsertSQLServer(BaseIncrementalPredicates): class TestPredicatesDeleteInsertSQLServer(BaseIncrementalPredicates): @pytest.fixture(scope="class") def project_config_update(self): - return {"models": {"+predicates": ["id != 2"], "+incremental_strategy": "delete+insert"}} + return { + "models": { + "+predicates": ["id != 2"], + "+incremental_strategy": "delete+insert", + } + } class TestBaseMergeExcludeColumnsSQLServer(BaseMergeExcludeColumns): diff --git a/tests/functional/adapter/test_persist_docs.py b/tests/functional/adapter/test_persist_docs.py index 83850620..c93d4a84 100644 --- a/tests/functional/adapter/test_persist_docs.py +++ b/tests/functional/adapter/test_persist_docs.py @@ -13,5 +13,7 @@ class TestPersistDocsColumnMissingSQLServer(BasePersistDocsColumnMissing): pass -class TestPersistDocsCommentOnQuotedColumnSQLServer(BasePersistDocsCommentOnQuotedColumn): +class TestPersistDocsCommentOnQuotedColumnSQLServer( + BasePersistDocsCommentOnQuotedColumn +): pass diff --git a/tests/functional/adapter/test_simple_copy.py b/tests/functional/adapter/test_simple_copy.py index b3b04108..9c6671cd 100644 --- a/tests/functional/adapter/test_simple_copy.py +++ b/tests/functional/adapter/test_simple_copy.py @@ -1,7 +1,10 @@ from dbt.tests.adapter.simple_copy.test_copy_uppercase import ( TestSimpleCopyUppercase as BaseSimpleCopyUppercase, ) -from dbt.tests.adapter.simple_copy.test_simple_copy import EmptyModelsArentRunBase, SimpleCopyBase +from dbt.tests.adapter.simple_copy.test_simple_copy import ( + EmptyModelsArentRunBase, + SimpleCopyBase, +) class TestSimpleCopyBaseSQLServer(SimpleCopyBase): diff --git a/tests/functional/adapter/test_simple_seed.py b/tests/functional/adapter/test_simple_seed.py index 8157566c..4077d5b9 100644 --- a/tests/functional/adapter/test_simple_seed.py +++ b/tests/functional/adapter/test_simple_seed.py @@ -1,13 +1,17 @@ import pytest from dbt.tests.adapter.simple_seed.seeds import seeds__expected_sql -from dbt.tests.adapter.simple_seed.test_seed import TestBasicSeedTests as BaseBasicSeedTests +from dbt.tests.adapter.simple_seed.test_seed import ( + TestBasicSeedTests as BaseBasicSeedTests, +) from dbt.tests.adapter.simple_seed.test_seed import ( TestSeedConfigFullRefreshOff as BaseSeedConfigFullRefreshOff, ) from dbt.tests.adapter.simple_seed.test_seed import ( TestSeedConfigFullRefreshOn as BaseSeedConfigFullRefreshOn, ) -from dbt.tests.adapter.simple_seed.test_seed import TestSeedCustomSchema as BaseSeedCustomSchema +from dbt.tests.adapter.simple_seed.test_seed import ( + TestSeedCustomSchema as BaseSeedCustomSchema, +) from dbt.tests.adapter.simple_seed.test_seed import TestSeedParsing as BaseSeedParsing from dbt.tests.adapter.simple_seed.test_seed import ( TestSeedSpecificFormats as BaseSeedSpecificFormats, @@ -24,9 +28,9 @@ from dbt.adapters.sqlserver import SQLServerAdapter -fixed_setup_sql = seeds__expected_sql.replace("TIMESTAMP WITHOUT TIME ZONE", "DATETIME").replace( - "TEXT", "VARCHAR(255)" -) +fixed_setup_sql = seeds__expected_sql.replace( + "TIMESTAMP WITHOUT TIME ZONE", "DATETIME" +).replace("TEXT", "VARCHAR(255)") seeds__tricky_csv = """ seed_id,seed_id_str,a_bool,looks_like_a_bool,a_date,looks_like_a_date,relative,weekday @@ -168,7 +172,9 @@ def clear_test_schema(self, project): adapter = project.adapter assert isinstance(project.adapter, SQLServerAdapter) with get_connection(project.adapter): - rel = adapter.Relation.create(database=project.database, schema=project.test_schema) + rel = adapter.Relation.create( + database=project.database, schema=project.test_schema + ) adapter.drop_schema(rel) diff --git a/tests/functional/adapter/test_simple_snapshot.py b/tests/functional/adapter/test_simple_snapshot.py index 5ef0c33d..9b5fc857 100644 --- a/tests/functional/adapter/test_simple_snapshot.py +++ b/tests/functional/adapter/test_simple_snapshot.py @@ -1,4 +1,7 @@ -from dbt.tests.adapter.simple_snapshot.test_snapshot import BaseSimpleSnapshot, BaseSnapshotCheck +from dbt.tests.adapter.simple_snapshot.test_snapshot import ( + BaseSimpleSnapshot, + BaseSnapshotCheck, +) class TestSnapshotSQLServer(BaseSimpleSnapshot): diff --git a/tests/functional/adapter/test_sources.py b/tests/functional/adapter/test_sources.py index 91c34db8..bae2c614 100644 --- a/tests/functional/adapter/test_sources.py +++ b/tests/functional/adapter/test_sources.py @@ -1,5 +1,8 @@ import pytest -from dbt.tests.adapter.basic.files import config_materialized_table, config_materialized_view +from dbt.tests.adapter.basic.files import ( + config_materialized_table, + config_materialized_view, +) from dbt.tests.util import run_dbt source_regular = """ diff --git a/tests/functional/adapter/test_utils.py b/tests/functional/adapter/test_utils.py index e1072473..93123f8a 100644 --- a/tests/functional/adapter/test_utils.py +++ b/tests/functional/adapter/test_utils.py @@ -1,5 +1,7 @@ import pytest -from dbt.tests.adapter.utils.fixture_cast_bool_to_text import models__test_cast_bool_to_text_yml +from dbt.tests.adapter.utils.fixture_cast_bool_to_text import ( + models__test_cast_bool_to_text_yml, +) from dbt.tests.adapter.utils.fixture_listagg import ( models__test_listagg_yml, seeds__data_listagg_csv, @@ -15,7 +17,9 @@ from dbt.tests.adapter.utils.test_date_trunc import BaseDateTrunc from dbt.tests.adapter.utils.test_dateadd import BaseDateAdd from dbt.tests.adapter.utils.test_datediff import BaseDateDiff -from dbt.tests.adapter.utils.test_escape_single_quotes import BaseEscapeSingleQuotesQuote +from dbt.tests.adapter.utils.test_escape_single_quotes import ( + BaseEscapeSingleQuotesQuote, +) from dbt.tests.adapter.utils.test_except import BaseExcept from dbt.tests.adapter.utils.test_hash import BaseHash from dbt.tests.adapter.utils.test_intersect import BaseIntersect diff --git a/tests/functional/adapter/test_utils_data_types.py b/tests/functional/adapter/test_utils_data_types.py index c091120d..e476ec0e 100644 --- a/tests/functional/adapter/test_utils_data_types.py +++ b/tests/functional/adapter/test_utils_data_types.py @@ -11,7 +11,9 @@ ) -@pytest.mark.skip(reason="SQL Server shows 'numeric' if you don't explicitly cast it to bigint") +@pytest.mark.skip( + reason="SQL Server shows 'numeric' if you don't explicitly cast it to bigint" +) class TestTypeBigIntSQLServer(BaseTypeBigInt): pass diff --git a/tests/unit/adapters/sqlserver/test_sql_server_connection_manager.py b/tests/unit/adapters/sqlserver/test_sql_server_connection_manager.py index b321da0c..9626adca 100644 --- a/tests/unit/adapters/sqlserver/test_sql_server_connection_manager.py +++ b/tests/unit/adapters/sqlserver/test_sql_server_connection_manager.py @@ -73,7 +73,8 @@ def test_get_pyodbc_attrs_before_contains_access_token_key_for_cli_authenticatio @pytest.mark.parametrize( - "key, value, expected", [("somekey", False, "somekey=No"), ("somekey", True, "somekey=Yes")] + "key, value, expected", + [("somekey", False, "somekey=No"), ("somekey", True, "somekey=Yes")], ) def test_bool_to_connection_string_arg(key: str, value: bool, expected: str) -> None: assert bool_to_connection_string_arg(key, value) == expected From 3f276672390a1826c93fc8d60e04359c3e64b98a Mon Sep 17 00:00:00 2001 From: Ty Schlichenmeyer Date: Wed, 1 Nov 2023 15:57:56 +0000 Subject: [PATCH 27/33] fix: specify odbc driver 18 --- .devcontainer/devcontainer.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 0e31f685..e1043d67 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -3,13 +3,16 @@ "image": "mcr.microsoft.com/devcontainers/python:3.11", "features": { "ghcr.io/devcontainers/features/docker-in-docker:2": {}, - "ghcr.io/jlaundry/devcontainer-features/mssql-odbc-driver:1": {}, + "ghcr.io/jlaundry/devcontainer-features/mssql-odbc-driver:1": { + "version": "18" + }, "ghcr.io/devcontainers/features/azure-cli:1": {} }, "postCreateCommand": "cp test.env.sample test.env && make dev", "customizations": { "vscode": { "extensions": [ + "ms-python.python", "ms-python.vscode-pylance", "ms-vscode.makefile-tools", "innoverio.vscode-dbt-power-user" From 4794cdf8432c10403776f8a3f567794c685f7a7b Mon Sep 17 00:00:00 2001 From: Ty Schlichenmeyer Date: Wed, 1 Nov 2023 16:07:54 +0000 Subject: [PATCH 28/33] ci(pytest): fail fast --- pytest.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/pytest.ini b/pytest.ini index a4e2011f..26707093 100644 --- a/pytest.ini +++ b/pytest.ini @@ -10,3 +10,4 @@ testpaths = markers = skip_profile only_with_profile +addopts = -x From 5df68e06fbb84d16f28e2a3c612d1bd7745e2915 Mon Sep 17 00:00:00 2001 From: Ty Schlichenmeyer Date: Wed, 1 Nov 2023 16:09:29 +0000 Subject: [PATCH 29/33] fix: broken module imports in tests --- tests/functional/adapter/test_incremental.py | 2 +- tests/functional/adapter/test_persist_docs.py | 2 +- tests/functional/adapter/test_relations.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/functional/adapter/test_incremental.py b/tests/functional/adapter/test_incremental.py index 132128ba..521e3023 100644 --- a/tests/functional/adapter/test_incremental.py +++ b/tests/functional/adapter/test_incremental.py @@ -20,7 +20,7 @@ BaseIncrementalUniqueKey, ) -from tests.adapter.dbt.tests.adapter.incremental.test_incremental_merge_exclude_columns import ( +from dbt.tests.adapter.incremental.test_incremental_merge_exclude_columns import ( BaseMergeExcludeColumns, ) diff --git a/tests/functional/adapter/test_persist_docs.py b/tests/functional/adapter/test_persist_docs.py index c93d4a84..62d50d9d 100644 --- a/tests/functional/adapter/test_persist_docs.py +++ b/tests/functional/adapter/test_persist_docs.py @@ -1,4 +1,4 @@ -from tests.adapter.dbt.tests.adapter.persist_docs.test_persist_docs import ( +from dbt.tests.adapter.persist_docs.test_persist_docs import ( BasePersistDocs, BasePersistDocsColumnMissing, BasePersistDocsCommentOnQuotedColumn, diff --git a/tests/functional/adapter/test_relations.py b/tests/functional/adapter/test_relations.py index 0673e69b..e273ae01 100644 --- a/tests/functional/adapter/test_relations.py +++ b/tests/functional/adapter/test_relations.py @@ -1,4 +1,4 @@ -from tests.adapter.dbt.tests.adapter.relations.test_changing_relation_type import ( +from dbt.tests.adapter.relations.test_changing_relation_type import ( BaseChangeRelationTypeValidator, ) From d256821dfb80a12b3f1fe41253e8d9a607c20219 Mon Sep 17 00:00:00 2001 From: Ty Schlichenmeyer Date: Wed, 1 Nov 2023 16:10:07 +0000 Subject: [PATCH 30/33] build: upgrade dbt-core to latest patch --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d2c34eb9..24da1bdb 100644 --- a/setup.py +++ b/setup.py @@ -68,7 +68,7 @@ def run(self): packages=find_namespace_packages(include=["dbt", "dbt.*"]), include_package_data=True, install_requires=[ - "dbt-core~=1.5.1rc1", + "dbt-core~=1.5.8", "pyodbc~=4.0.35,!=4.0.36,!=4.0.37", "azure-identity>=1.12.0", ], From 94440629690ea0d7bd80251cdaddca3488c2c517 Mon Sep 17 00:00:00 2001 From: Ty Schlichenmeyer Date: Wed, 1 Nov 2023 16:11:02 +0000 Subject: [PATCH 31/33] ci: initialize pdm project --- .pdm-python | 1 + pdm.lock | 18 +++++++++++++++ pyproject.toml | 43 +++++++++++++++++++++++++++++++++++ src/dbt_sqlserver/__init__.py | 0 4 files changed, 62 insertions(+) create mode 100644 .pdm-python create mode 100644 pdm.lock create mode 100644 pyproject.toml create mode 100644 src/dbt_sqlserver/__init__.py diff --git a/.pdm-python b/.pdm-python new file mode 100644 index 00000000..ddadb4e9 --- /dev/null +++ b/.pdm-python @@ -0,0 +1 @@ +/workspaces/dbt-sqlserver/.venv/bin/python diff --git a/pdm.lock b/pdm.lock new file mode 100644 index 00000000..0733ade5 --- /dev/null +++ b/pdm.lock @@ -0,0 +1,18 @@ +# This file is @generated by PDM. +# It is not intended for manual editing. + +[metadata] +groups = ["default"] +strategy = ["cross_platform"] +lock_version = "4.4" +content_hash = "sha256:344e9100ba537f997f93b5e6fb2095644f7c60d610097daa9a91c7c38193e345" + +[[package]] +name = "setuptools" +version = "68.2.2" +requires_python = ">=3.8" +summary = "Easily download, build, install, upgrade, and uninstall Python packages" +files = [ + {file = "setuptools-68.2.2-py3-none-any.whl", hash = "sha256:b454a35605876da60632df1a60f736524eb73cc47bbc9f3f1ef1b644de74fd2a"}, + {file = "setuptools-68.2.2.tar.gz", hash = "sha256:4ac1475276d2f1c48684874089fefcd83bd7162ddaafb81fac866ba0db282a87"}, +] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..ba6ed1e4 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,43 @@ +[project] +name = "dbt-sqlserver" +version = "1.5.0" +description = "A Microsoft SQL Server adapter plugin for dbt" +authors = [ + {name = "Mikael Ene"}, + {name = "Anders Swanson"}, + {name = "Sam Debruyn"}, + {name = "Cor Zuurmond"}, + {name = "Ty Schlichenmeyer", email="ty.schlich@gmail.com"} +] +dependencies = [ + "setuptools>=68.2.2", + "azure-identity>=1.12.0", + "dbt-core~=1.5.1rc1", + "pyodbc~=4.0.35,!=4.0.36,!=4.0.37", +] +requires-python = ">=3.11" +readme = "README.md" +license = {text = "MIT"} +classifiers = [ + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: MIT License", + "Operating System :: MacOS :: MacOS X", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", +] + +[project.urls] +Homepage = "https://github.com/dbt-msft/dbt-sqlserver" +"Setup & configuration" = "https://docs.getdbt.com/reference/warehouse-profiles/mssql-profile" +"Documentation & usage" = "https://docs.getdbt.com/reference/resource-configs/mssql-configs" +Changelog = "https://github.com/dbt-msft/dbt-sqlserver/blob/master/CHANGELOG.md" +"Issue Tracker" = "https://github.com/dbt-msft/dbt-sqlserver/issues" + +[build-system] +requires = ["pdm-backend"] +build-backend = "pdm.backend" diff --git a/src/dbt_sqlserver/__init__.py b/src/dbt_sqlserver/__init__.py new file mode 100644 index 00000000..e69de29b From 0e185fcca7b5e85d6e5392ca472198be4e0d9c27 Mon Sep 17 00:00:00 2001 From: Ty Schlichenmeyer Date: Sat, 18 Nov 2023 07:13:41 -0600 Subject: [PATCH 32/33] checkpoint --- .devcontainer/devcontainer.json | 61 +++++--- .pdm-python | 1 - .pre-commit-config.yaml | 13 -- .vscode/settings.json | 7 - CONTRIBUTING.md | 6 +- Makefile | 4 +- {src/dbt_sqlserver => dbt}/__init__.py | 0 dbt/adapters/sqlserver/__version__.py | 2 +- dev_requirements.txt | 153 +++++++++++++++++-- devops/CI.Dockerfile | 3 +- pdm.lock | 18 --- pyproject.toml | 54 ++++--- pytest.ini | 13 -- setup.py | 17 +-- tests/functional/adapter/test_basic.py | 96 +++++------- tests/functional/adapter/test_simple_seed.py | 2 + 16 files changed, 278 insertions(+), 172 deletions(-) delete mode 100644 .pdm-python delete mode 100644 .vscode/settings.json rename {src/dbt_sqlserver => dbt}/__init__.py (100%) delete mode 100644 pdm.lock delete mode 100644 pytest.ini diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index e1043d67..4e08570c 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,22 +1,47 @@ { - "name": "dbt-sqlserver", - "image": "mcr.microsoft.com/devcontainers/python:3.11", - "features": { - "ghcr.io/devcontainers/features/docker-in-docker:2": {}, - "ghcr.io/jlaundry/devcontainer-features/mssql-odbc-driver:1": { - "version": "18" - }, - "ghcr.io/devcontainers/features/azure-cli:1": {} + "name": "dbt-sqlserver", + "image": "mcr.microsoft.com/devcontainers/python:3.12", + "features": { + "ghcr.io/devcontainers/features/docker-in-docker:2": {}, + "ghcr.io/jlaundry/devcontainer-features/mssql-odbc-driver:1": { + "version": "18" }, - "postCreateCommand": "cp test.env.sample test.env && make dev", - "customizations": { - "vscode": { - "extensions": [ - "ms-python.python", - "ms-python.vscode-pylance", - "ms-vscode.makefile-tools", - "innoverio.vscode-dbt-power-user" - ] - } + "ghcr.io/devcontainers/features/azure-cli:1": {} + }, + "customizations": { + "vscode": { + "settings": { + "python.defaultInterpreterPath": ".venv/bin/python", + "python.terminal.activateEnvInCurrentTerminal": true, + "[python]": { + "editor.codeActionsOnSave": { + "source.fixAll": true, + "source.organizeImports": true + }, + "editor.defaultFormatter": "charliermarsh.ruff" + }, + "python.testing.cwd": "tests", + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true, + "editor.formatOnSave": true, + "editor.formatOnPaste": true, + "editor.stickyScroll.enabled": true, + "dbt.enableNewLineagePanel": true, + "files.insertFinalNewline": true, + "terminal.integrated.environmentChangesIndicator": "on" + }, + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance", + "innoverio.vscode-dbt-power-user", + "ms-vscode.makefile-tools", + "charliermarsh.ruff", + "eamodio.gitlens", + "GitHub.vscode-github-actions", + "tamasfe.even-better-toml", + "esbenp.prettier-vscode", + "redhat.vscode-yaml" + ] } + } } diff --git a/.pdm-python b/.pdm-python deleted file mode 100644 index ddadb4e9..00000000 --- a/.pdm-python +++ /dev/null @@ -1 +0,0 @@ -/workspaces/dbt-sqlserver/.venv/bin/python diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 60c90944..a89196fa 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,5 @@ default_language_version: python: python3.11 - repos: - repo: 'https://github.com/pre-commit/pre-commit-hooks' rev: v4.4.0 @@ -30,18 +29,6 @@ repos: - '-d {extends: default, rules: {line-length: disable, document-start: disable}}' - '-s' - - repo: 'https://github.com/MarcoGorelli/absolufy-imports' - rev: v0.3.1 - hooks: - - id: absolufy-imports - - - repo: 'https://github.com/hadialqattan/pycln' - rev: v2.1.3 - hooks: - - id: pycln - args: - - '--all' - - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.1.3 hooks: diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index a3a18383..00000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "python.testing.pytestArgs": [ - "tests" - ], - "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true -} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4154aa19..45d493f6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,13 +4,13 @@ A recent version of [Docker](https://docs.docker.com/get-docker/). -Python 3.10 +Python 3.11 Create a virtual environment to properly isolate your Python dependencies. Here is an example of how to do so using the [pyenv](https://github.com/pyenv/pyenv) CLI: ```shell -pyenv install 3.10.7 -pyenv virtualenv 3.10.7 dbt-sqlserver +pyenv install 3.11.6 +pyenv virtualenv 3.11.6 dbt-sqlserver pyenv activate dbt-sqlserver ``` diff --git a/Makefile b/Makefile index c98cc416..513595d2 100644 --- a/Makefile +++ b/Makefile @@ -39,12 +39,12 @@ linecheck: ## Checks for all Python lines 100 characters or more .PHONY: unit unit: ## Runs unit tests. @\ - pytest -n auto -ra -v tests/unit + pytest tests/unit .PHONY: functional functional: ## Runs functional tests. @\ - pytest -n auto -ra -v tests/functional + pytest tests/functional .PHONY: test test: ## Runs unit tests and code checks against staged changes. diff --git a/src/dbt_sqlserver/__init__.py b/dbt/__init__.py similarity index 100% rename from src/dbt_sqlserver/__init__.py rename to dbt/__init__.py diff --git a/dbt/adapters/sqlserver/__version__.py b/dbt/adapters/sqlserver/__version__.py index e3a0f015..86531afb 100644 --- a/dbt/adapters/sqlserver/__version__.py +++ b/dbt/adapters/sqlserver/__version__.py @@ -1 +1 @@ -version = "1.5.0" +version = "1.5.9" diff --git a/dev_requirements.txt b/dev_requirements.txt index 97ac7bbb..8225a7b7 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,10 +1,143 @@ -pytest==7.3.1 -twine==4.0.2 -wheel==0.40.0 -pre-commit==2.21.0;python_version<"3.8" -pre-commit==3.3.2;python_version>="3.8" -pytest-dotenv==0.5.2 -dbt-tests-adapter~=1.5.1rc1 -flaky==3.7.0 -pytest-xdist==3.3.1 --e . +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --extra=dev --output-file=dev-requirements.txt pyproject.toml +# +agate==1.7.0 + # via dbt-core +attrs==23.1.0 + # via + # jsonschema + # referencing +azure-core==1.29.5 + # via azure-identity +azure-identity==1.15.0 + # via dbt-sqlserver (pyproject.toml) +babel==2.13.1 + # via agate +certifi==2023.7.22 + # via requests +cffi==1.16.0 + # via + # cryptography + # dbt-core +charset-normalizer==3.3.2 + # via requests +click==8.1.7 + # via dbt-core +colorama==0.4.6 + # via dbt-core +cryptography==41.0.5 + # via + # azure-identity + # msal + # pyjwt +dbt-core==1.5.9 + # via dbt-sqlserver (pyproject.toml) +dbt-extractor==0.4.1 + # via dbt-core +future==0.18.3 + # via parsedatetime +hologram==0.0.16 + # via dbt-core +idna==3.4 + # via + # dbt-core + # requests +isodate==0.6.1 + # via + # agate + # dbt-core +jinja2==3.1.2 + # via dbt-core +jsonschema==4.19.2 + # via hologram +jsonschema-specifications==2023.11.1 + # via jsonschema +leather==0.3.4 + # via agate +logbook==1.5.3 + # via dbt-core +markupsafe==2.1.3 + # via + # jinja2 + # werkzeug +mashumaro[msgpack]==3.6 + # via + # dbt-core + # mashumaro +minimal-snowplow-tracker==0.0.2 + # via dbt-core +msal==1.25.0 + # via + # azure-identity + # msal-extensions +msal-extensions==1.0.0 + # via azure-identity +msgpack==1.0.7 + # via mashumaro +networkx==2.8.8 + # via dbt-core +packaging==23.2 + # via dbt-core +parsedatetime==2.4 + # via agate +pathspec==0.11.2 + # via dbt-core +portalocker==2.8.2 + # via msal-extensions +protobuf==4.25.1 + # via dbt-core +pycparser==2.21 + # via cffi +pyjwt[crypto]==2.8.0 + # via + # msal + # pyjwt +pyodbc==5.0.1 + # via dbt-sqlserver (pyproject.toml) +python-dateutil==2.8.2 + # via hologram +python-slugify==8.0.1 + # via agate +pytimeparse==1.1.8 + # via agate +pytz==2023.3.post1 + # via dbt-core +pyyaml==6.0.1 + # via dbt-core +referencing==0.31.0 + # via + # jsonschema + # jsonschema-specifications +requests==2.31.0 + # via + # azure-core + # dbt-core + # minimal-snowplow-tracker + # msal +rpds-py==0.13.0 + # via + # jsonschema + # referencing +six==1.16.0 + # via + # azure-core + # isodate + # leather + # minimal-snowplow-tracker + # python-dateutil +sqlparse==0.4.4 + # via dbt-core +text-unidecode==1.3 + # via python-slugify +typing-extensions==4.8.0 + # via + # azure-core + # dbt-core + # mashumaro +urllib3==2.1.0 + # via requests +werkzeug==2.3.8 + # via dbt-core diff --git a/devops/CI.Dockerfile b/devops/CI.Dockerfile index 09e95f9a..ce9beb85 100644 --- a/devops/CI.Dockerfile +++ b/devops/CI.Dockerfile @@ -1,4 +1,4 @@ -ARG PYTHON_VERSION="3.10" +ARG PYTHON_VERSION="3.11" FROM python:${PYTHON_VERSION}-bullseye as base # Setup dependencies for pyodbc @@ -7,6 +7,7 @@ RUN apt-get update && \ apt-transport-https \ curl \ gnupg2 \ + unixodbc \ unixodbc-dev \ lsb-release && \ apt-get autoremove -yqq --purge && \ diff --git a/pdm.lock b/pdm.lock deleted file mode 100644 index 0733ade5..00000000 --- a/pdm.lock +++ /dev/null @@ -1,18 +0,0 @@ -# This file is @generated by PDM. -# It is not intended for manual editing. - -[metadata] -groups = ["default"] -strategy = ["cross_platform"] -lock_version = "4.4" -content_hash = "sha256:344e9100ba537f997f93b5e6fb2095644f7c60d610097daa9a91c7c38193e345" - -[[package]] -name = "setuptools" -version = "68.2.2" -requires_python = ">=3.8" -summary = "Easily download, build, install, upgrade, and uninstall Python packages" -files = [ - {file = "setuptools-68.2.2-py3-none-any.whl", hash = "sha256:b454a35605876da60632df1a60f736524eb73cc47bbc9f3f1ef1b644de74fd2a"}, - {file = "setuptools-68.2.2.tar.gz", hash = "sha256:4ac1475276d2f1c48684874089fefcd83bd7162ddaafb81fac866ba0db282a87"}, -] diff --git a/pyproject.toml b/pyproject.toml index ba6ed1e4..054a3bf0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,35 +1,46 @@ [project] name = "dbt-sqlserver" -version = "1.5.0" +dynamic = ["version"] description = "A Microsoft SQL Server adapter plugin for dbt" +readme = "README.md" +license = "MIT" authors = [ - {name = "Mikael Ene"}, - {name = "Anders Swanson"}, - {name = "Sam Debruyn"}, - {name = "Cor Zuurmond"}, - {name = "Ty Schlichenmeyer", email="ty.schlich@gmail.com"} -] -dependencies = [ - "setuptools>=68.2.2", - "azure-identity>=1.12.0", - "dbt-core~=1.5.1rc1", - "pyodbc~=4.0.35,!=4.0.36,!=4.0.37", + { name = "Mikael Ene" }, + { name = "Anders Swanson" }, + { name = "Sam Debruyn" }, + { name = "Cor Zuurmond" }, + { name = "Ty Schlichenmeyer" }, ] -requires-python = ">=3.11" -readme = "README.md" -license = {text = "MIT"} classifiers = [ "Development Status :: 5 - Production/Stable", "License :: OSI Approved :: MIT License", "Operating System :: MacOS :: MacOS X", "Operating System :: Microsoft :: Windows", "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ] +dependencies = ["azure-identity", "dbt-core<1.6", "pyodbc"] + +[project.optional-dependencies] +test = ["pytest", "ruff", "pre-commit", "dbt-tests-adapter<1.6"] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.version] +path = "dbt/adapters/sqlserver/__version__.py" + +[tool.hatch.build] +include = ["dbt", "dbt.include.sqlserver", "dbt.adapters.sqlserver"] + +[tool.hatch.envs.test] +dependencies = ["pre-commit", "pytest", "ruff"] + [project.urls] Homepage = "https://github.com/dbt-msft/dbt-sqlserver" @@ -38,6 +49,9 @@ Homepage = "https://github.com/dbt-msft/dbt-sqlserver" Changelog = "https://github.com/dbt-msft/dbt-sqlserver/blob/master/CHANGELOG.md" "Issue Tracker" = "https://github.com/dbt-msft/dbt-sqlserver/issues" -[build-system] -requires = ["pdm-backend"] -build-backend = "pdm.backend" + +[tool.pytest.ini_options] +env_files = ["test.env"] +testpaths = ["tests/unit", "tests/functional"] +markers = ["skip_profile", "only_with_profile"] +addopts = "-x" diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 26707093..00000000 --- a/pytest.ini +++ /dev/null @@ -1,13 +0,0 @@ -[pytest] -filterwarnings = - ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning - ignore:unclosed file .*:ResourceWarning -env_files = - test.env -testpaths = - tests/unit - tests/functional -markers = - skip_profile - only_with_profile -addopts = -x diff --git a/setup.py b/setup.py index 24da1bdb..8b881b75 100644 --- a/setup.py +++ b/setup.py @@ -7,8 +7,14 @@ from setuptools.command.install import install package_name = "dbt-sqlserver" -authors_list = ["Mikael Ene", "Anders Swanson", "Sam Debruyn", "Cor Zuurmond"] -dbt_version = "1.5" +authors_list = [ + "Mikael Ene", + "Anders Swanson", + "Sam Debruyn", + "Cor Zuurmond", + "Ty Schlichenmeyer", +] +dbt_version = "1.5.9" description = """A Microsoft SQL Server adapter plugin for dbt""" this_directory = os.path.abspath(os.path.dirname(__file__)) @@ -65,13 +71,6 @@ def run(self): license="MIT", author=", ".join(authors_list), url="https://github.com/dbt-msft/dbt-sqlserver", - packages=find_namespace_packages(include=["dbt", "dbt.*"]), - include_package_data=True, - install_requires=[ - "dbt-core~=1.5.8", - "pyodbc~=4.0.35,!=4.0.36,!=4.0.37", - "azure-identity>=1.12.0", - ], cmdclass={ "verify": VerifyVersionCommand, }, diff --git a/tests/functional/adapter/test_basic.py b/tests/functional/adapter/test_basic.py index 0efcc5a3..c014954c 100644 --- a/tests/functional/adapter/test_basic.py +++ b/tests/functional/adapter/test_basic.py @@ -1,99 +1,83 @@ import os import pytest -from dbt.tests.adapter.basic.expected_catalog import ( - base_expected_catalog, - expected_references_catalog, - no_stats, +from dbt.tests.adapter.basic import ( + expected_catalog, + files, + test_base, + test_adapter_methods, + test_docs_generate, + test_empty, + test_ephemeral, + test_generic_tests, + test_incremental , + test_singular_tests, + test_singular_tests_ephemeral, + test_snapshot_check_cols, + test_snapshot_timestamp, + test_table_materialization, + test_validate_connection, ) -from dbt.tests.adapter.basic.files import incremental_not_schema_change_sql -from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod -from dbt.tests.adapter.basic.test_base import BaseSimpleMaterializations -from dbt.tests.adapter.basic.test_docs_generate import ( - BaseDocsGenerate, - BaseDocsGenReferences, - ref_models__docs_md, - ref_models__ephemeral_copy_sql, - ref_models__schema_yml, - ref_sources__schema_yml, -) -from dbt.tests.adapter.basic.test_empty import BaseEmpty -from dbt.tests.adapter.basic.test_ephemeral import BaseEphemeral -from dbt.tests.adapter.basic.test_generic_tests import BaseGenericTests -from dbt.tests.adapter.basic.test_incremental import ( - BaseIncremental, - BaseIncrementalNotSchemaChange, -) -from dbt.tests.adapter.basic.test_singular_tests import BaseSingularTests -from dbt.tests.adapter.basic.test_singular_tests_ephemeral import ( - BaseSingularTestsEphemeral, -) -from dbt.tests.adapter.basic.test_snapshot_check_cols import BaseSnapshotCheckCols -from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp -from dbt.tests.adapter.basic.test_table_materialization import BaseTableMaterialization -from dbt.tests.adapter.basic.test_validate_connection import BaseValidateConnection - -class TestSimpleMaterializationsSQLServer(BaseSimpleMaterializations): +class TestSimpleMaterializationsSQLServer(test_base.BaseSimpleMaterializations): pass -class TestSingularTestsSQLServer(BaseSingularTests): +class TestSingularTestsSQLServer(test_singular_tests.BaseSingularTests): pass @pytest.mark.skip(reason="ephemeral not supported") -class TestSingularTestsEphemeralSQLServer(BaseSingularTestsEphemeral): +class TestSingularTestsEphemeralSQLServer(test_singular_tests_ephemeral.BaseSingularTestsEphemeral): pass -class TestEmptySQLServer(BaseEmpty): +class TestEmptySQLServer(test_empty.BaseEmpty): pass - -class TestEphemeralSQLServer(BaseEphemeral): +class TestEphemeralSQLServer(test_ephemeral.BaseEphemeral): pass -class TestIncrementalSQLServer(BaseIncremental): +class TestIncrementalSQLServer(test_incremental.BaseIncremental): pass -class TestIncrementalNotSchemaChangeSQLServer(BaseIncrementalNotSchemaChange): +class TestIncrementalNotSchemaChangeSQLServer(test_incremental.BaseIncrementalNotSchemaChange): @pytest.fixture(scope="class") def models(self): return { - "incremental_not_schema_change.sql": incremental_not_schema_change_sql.replace( + "incremental_not_schema_change.sql": files.incremental_not_schema_change_sql.replace( "||", "+" ) } -class TestGenericTestsSQLServer(BaseGenericTests): +class TestGenericTestsSQLServer(test_generic_tests.BaseGenericTests): pass -class TestSnapshotCheckColsSQLServer(BaseSnapshotCheckCols): +class TestSnapshotCheckColsSQLServer(test_snapshot_check_cols.BaseSnapshotCheckCols): pass -class TestSnapshotTimestampSQLServer(BaseSnapshotTimestamp): +class TestSnapshotTimestampSQLServer(test_snapshot_timestamp.BaseSnapshotTimestamp): pass -class TestBaseCachingSQLServer(BaseAdapterMethod): +class TestBaseCachingSQLServer(test_adapter_methods.BaseAdapterMethod): pass -class TestValidateConnectionSQLServer(BaseValidateConnection): +class TestValidateConnectionSQLServer(test_validate_connection.BaseValidateConnection): pass -class TestTableMatSQLServer(BaseTableMaterialization): +class TestTableMatSQLServer(test_table_materialization.BaseTableMaterialization): pass -class TestDocsGenerateSQLServer(BaseDocsGenerate): +class TestDocsGenerateSQLServer(test_docs_generate.BaseDocsGenerate): @staticmethod @pytest.fixture(scope="class") def dbt_profile_target_update(): @@ -101,7 +85,7 @@ def dbt_profile_target_update(): @pytest.fixture(scope="class") def expected_catalog(self, project): - return base_expected_catalog( + return expected_catalog.base_expected_catalog( project, role=os.getenv("DBT_TEST_USER_1"), id_type="int", @@ -109,11 +93,11 @@ def expected_catalog(self, project): time_type="datetime", view_type="VIEW", table_type="BASE TABLE", - model_stats=no_stats(), + model_stats=expected_catalog.no_stats(), ) -class TestDocsGenReferencesSQLServer(BaseDocsGenReferences): +class TestDocsGenReferencesSQLServer(test_docs_generate.BaseDocsGenReferences): @staticmethod @pytest.fixture(scope="class") def dbt_profile_target_update(): @@ -121,7 +105,7 @@ def dbt_profile_target_update(): @pytest.fixture(scope="class") def expected_catalog(self, project): - return expected_references_catalog( + return expected_catalog.expected_references_catalog( project, role=os.getenv("DBT_TEST_USER_1"), id_type="int", @@ -130,7 +114,7 @@ def expected_catalog(self, project): bigint_type="int", view_type="VIEW", table_type="BASE TABLE", - model_stats=no_stats(), + model_stats=expected_catalog.no_stats(), ) @pytest.fixture(scope="class") @@ -157,12 +141,12 @@ def models(self): """ return { - "schema.yml": ref_models__schema_yml, - "sources.yml": ref_sources__schema_yml, + "schema.yml": test_docs_generate.ref_models__schema_yml, + "sources.yml": test_docs_generate.ref_sources__schema_yml, # order by not allowed in VIEWS "view_summary.sql": ref_models__view_summary_sql_no_order_by, # order by not allowed in CTEs "ephemeral_summary.sql": ref_models__ephemeral_summary_sql_no_order_by, - "ephemeral_copy.sql": ref_models__ephemeral_copy_sql, - "docs.md": ref_models__docs_md, + "ephemeral_copy.sql": test_docs_generate.ref_models__ephemeral_copy_sql, + "docs.md": test_docs_generate.ref_models__docs_md, } diff --git a/tests/functional/adapter/test_simple_seed.py b/tests/functional/adapter/test_simple_seed.py index 4077d5b9..ff185277 100644 --- a/tests/functional/adapter/test_simple_seed.py +++ b/tests/functional/adapter/test_simple_seed.py @@ -1,4 +1,6 @@ import pytest + + from dbt.tests.adapter.simple_seed.seeds import seeds__expected_sql from dbt.tests.adapter.simple_seed.test_seed import ( TestBasicSeedTests as BaseBasicSeedTests, From ecdda8b580f8736b588680df2c08a0e86349db01 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 18 Nov 2023 13:16:59 +0000 Subject: [PATCH 33/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/functional/adapter/test_basic.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/functional/adapter/test_basic.py b/tests/functional/adapter/test_basic.py index c014954c..4f452940 100644 --- a/tests/functional/adapter/test_basic.py +++ b/tests/functional/adapter/test_basic.py @@ -10,7 +10,7 @@ test_empty, test_ephemeral, test_generic_tests, - test_incremental , + test_incremental, test_singular_tests, test_singular_tests_ephemeral, test_snapshot_check_cols, @@ -19,6 +19,7 @@ test_validate_connection, ) + class TestSimpleMaterializationsSQLServer(test_base.BaseSimpleMaterializations): pass @@ -28,13 +29,16 @@ class TestSingularTestsSQLServer(test_singular_tests.BaseSingularTests): @pytest.mark.skip(reason="ephemeral not supported") -class TestSingularTestsEphemeralSQLServer(test_singular_tests_ephemeral.BaseSingularTestsEphemeral): +class TestSingularTestsEphemeralSQLServer( + test_singular_tests_ephemeral.BaseSingularTestsEphemeral +): pass class TestEmptySQLServer(test_empty.BaseEmpty): pass + class TestEphemeralSQLServer(test_ephemeral.BaseEphemeral): pass @@ -43,7 +47,9 @@ class TestIncrementalSQLServer(test_incremental.BaseIncremental): pass -class TestIncrementalNotSchemaChangeSQLServer(test_incremental.BaseIncrementalNotSchemaChange): +class TestIncrementalNotSchemaChangeSQLServer( + test_incremental.BaseIncrementalNotSchemaChange +): @pytest.fixture(scope="class") def models(self): return {