From 6a35e980b3018e9103cdb7bd71430ace98b78fbf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 09:41:03 -0700 Subject: [PATCH 1/9] [Backport 1.8.latest] quoting config not working with 1.8 (#1076) * quoting config not working with 1.8 (#1075) * revert change for quoting policy issue * make revet in both places * update format to ignore identifier name * update test to use SnowflakeRelation and render out the schema_relation, update macros to have condtion to check based on type * remove unneeded comments from updating test * add test based on quoting schema names * update name to more generic my_model (cherry picked from commit 57285d57d910b965dc0c769398d1c83e0f537bcc) * local pre-commit run --------- Co-authored-by: Matthew McKnight <91097623+McKnight-42@users.noreply.github.com> Co-authored-by: McKnight-42 --- .../unreleased/Fixes-20240607-102708.yaml | 6 +++++ dbt/include/snowflake/macros/adapters.sql | 27 +++++++++++++------ .../list_relations_tests/test_pagination.py | 16 +++++------ .../test_special_characters.py | 24 +++++++++++++++++ 4 files changed, 56 insertions(+), 17 deletions(-) create mode 100644 .changes/unreleased/Fixes-20240607-102708.yaml create mode 100644 tests/functional/adapter/list_relations_tests/test_special_characters.py diff --git a/.changes/unreleased/Fixes-20240607-102708.yaml b/.changes/unreleased/Fixes-20240607-102708.yaml new file mode 100644 index 000000000..58cd9bbee --- /dev/null +++ b/.changes/unreleased/Fixes-20240607-102708.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: return to previous naming convention to return to quoting policy +time: 2024-06-07T10:27:08.542159-05:00 +custom: + Author: McKnight-42 + Issue: "1074" diff --git a/dbt/include/snowflake/macros/adapters.sql b/dbt/include/snowflake/macros/adapters.sql index 0bf7b7d1b..6e7ea8f6c 100644 --- a/dbt/include/snowflake/macros/adapters.sql +++ b/dbt/include/snowflake/macros/adapters.sql @@ -72,9 +72,15 @@ {% for _ in range(0, max_iter) %} - {%- set paginated_sql -%} - show objects in {{ schema_relation.database }}.{{ schema_relation.schema }} limit {{ max_results_per_iter }} from '{{ watermark.table_name }}' - {%- endset -%} + {% if schema_relation is string %} + {%- set paginated_sql -%} + show objects in {{ schema_relation }} limit {{ max_results_per_iter }} from '{{ watermark.table_name }}' + {%- endset -%} + {% else %} + {%- set paginated_sql -%} + show objects in {{ schema_relation.include(identifier=False) }} limit {{ max_results_per_iter }} from '{{ watermark.table_name }}' + {%- endset -%} + {% endif -%} {%- set paginated_result = run_query(paginated_sql) %} {%- set paginated_n = (paginated_result | length) -%} @@ -96,7 +102,7 @@ {%- if loop.index == max_iter -%} {%- set msg -%} - dbt will list a maximum of {{ max_total_results }} objects in schema {{ schema_relation.database }}.{{ schema_relation.schema }}. + dbt will list a maximum of {{ max_total_results }} objects in schema {{ schema_relation }}. Your schema exceeds this limit. Please contact support@getdbt.com for troubleshooting tips, or review and reduce the number of objects contained. {%- endset -%} @@ -122,10 +128,15 @@ {% macro snowflake__list_relations_without_caching(schema_relation, max_iter=10, max_results_per_iter=10000) %} {%- set max_total_results = max_results_per_iter * max_iter -%} - - {%- set sql -%} - show objects in {{ schema_relation.database }}.{{ schema_relation.schema }} limit {{ max_results_per_iter }} - {%- endset -%} + {% if schema_relation is string %} + {%- set sql -%} + show objects in {{ schema_relation }} limit {{ max_results_per_iter }} + {%- endset -%} + {% else %} + {%- set sql -%} + show objects in {{ schema_relation.include(identifier=False) }} limit {{ max_results_per_iter }} + {%- endset -%} + {% endif -%} {%- set result = run_query(sql) -%} diff --git a/tests/functional/adapter/list_relations_tests/test_pagination.py b/tests/functional/adapter/list_relations_tests/test_pagination.py index 8f14a0012..407f9c501 100644 --- a/tests/functional/adapter/list_relations_tests/test_pagination.py +++ b/tests/functional/adapter/list_relations_tests/test_pagination.py @@ -1,9 +1,8 @@ import os - import pytest - import json from dbt.tests.util import run_dbt, run_dbt_and_capture +from dbt.adapters.snowflake import SnowflakeRelation # Ensure this is the correct import path # Testing rationale: # - snowflake SHOW TERSE OBJECTS command returns at max 10K objects in a single call @@ -122,8 +121,8 @@ def test__snowflake__list_relations_without_caching_termination(self, project): schemas = project.created_schemas for schema in schemas: - schema_relation = {"database": database, "schema": schema} - kwargs = {"schema_relation": schema_relation} + schema_relation = SnowflakeRelation.create(database=database, schema=schema) + kwargs = {"schema_relation": schema_relation.render()} _, log_output = run_dbt_and_capture( [ "--debug", @@ -137,7 +136,6 @@ def test__snowflake__list_relations_without_caching_termination(self, project): parsed_logs = parse_json_logs(log_output) n_relations = find_result_in_parsed_logs(parsed_logs, "n_relations") - assert n_relations == "n_relations: 1" @@ -171,8 +169,8 @@ def test__snowflake__list_relations_without_caching(self, project): schemas = project.created_schemas for schema in schemas: - schema_relation = {"database": database, "schema": schema} - kwargs = {"schema_relation": schema_relation} + schema_relation = SnowflakeRelation.create(database=database, schema=schema) + kwargs = {"schema_relation": schema_relation.render()} _, log_output = run_dbt_and_capture( [ "--debug", @@ -199,9 +197,9 @@ def test__snowflake__list_relations_without_caching_raise_error(self, project): schemas = project.created_schemas for schema in schemas: - schema_relation = {"database": database, "schema": schema} + schema_relation = SnowflakeRelation.create(database=database, schema=schema) - kwargs = {"schema_relation": schema_relation} + kwargs = {"schema_relation": schema_relation.render()} _, log_output = run_dbt_and_capture( [ "--debug", diff --git a/tests/functional/adapter/list_relations_tests/test_special_characters.py b/tests/functional/adapter/list_relations_tests/test_special_characters.py new file mode 100644 index 000000000..54c00962c --- /dev/null +++ b/tests/functional/adapter/list_relations_tests/test_special_characters.py @@ -0,0 +1,24 @@ +import pytest +from dbt.tests.util import run_dbt + + +TABLE_BASE_SQL = """ +-- models/my_model.sql +{{ config(schema = '1_contains_special*character$') }} +select 1 as id +""" + + +class TestSpecialCharactersInSchema: + @pytest.fixture(scope="class") + def project_config_update(self): + return {"quoting": {"schema": True}} + + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": TABLE_BASE_SQL, + } + + def test_schema_with_special_chars(self, project): + run_dbt(["run", "-s", "my_model"]) From d931f7aabf5dacff87cbaeb377baa6c8e4e76757 Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Wed, 12 Jun 2024 16:22:45 +0000 Subject: [PATCH 2/9] Bumping version to 1.8.3 and generate changelog --- .bumpversion.cfg | 2 +- .changes/1.8.3.md | 5 +++++ .changes/unreleased/Fixes-20240607-102708.yaml | 6 ------ CHANGELOG.md | 10 ++++++++-- dbt/adapters/snowflake/__version__.py | 2 +- 5 files changed, 15 insertions(+), 10 deletions(-) create mode 100644 .changes/1.8.3.md delete mode 100644 .changes/unreleased/Fixes-20240607-102708.yaml diff --git a/.bumpversion.cfg b/.bumpversion.cfg index ea067d8bd..22b3437af 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.8.2 +current_version = 1.8.3 parse = (?P[\d]+) # major version number \.(?P[\d]+) # minor version number \.(?P[\d]+) # patch version number diff --git a/.changes/1.8.3.md b/.changes/1.8.3.md new file mode 100644 index 000000000..09f89ef16 --- /dev/null +++ b/.changes/1.8.3.md @@ -0,0 +1,5 @@ +## dbt-snowflake 1.8.3 - June 12, 2024 + +### Fixes + +- return to previous naming convention to return to quoting policy ([#1074](https://github.com/dbt-labs/dbt-snowflake/issues/1074)) diff --git a/.changes/unreleased/Fixes-20240607-102708.yaml b/.changes/unreleased/Fixes-20240607-102708.yaml deleted file mode 100644 index 58cd9bbee..000000000 --- a/.changes/unreleased/Fixes-20240607-102708.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: return to previous naming convention to return to quoting policy -time: 2024-06-07T10:27:08.542159-05:00 -custom: - Author: McKnight-42 - Issue: "1074" diff --git a/CHANGELOG.md b/CHANGELOG.md index 19c0ab3ae..4e026c5c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,13 +5,19 @@ - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version. - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-snowflake/blob/main/CONTRIBUTING.md#adding-changelog-entry) -## dbt-snowflake 1.8.2 - May 23, 2024 +## dbt-snowflake 1.8.3 - June 12, 2024 ### Fixes -- Update relation caching to correctly identify dynamic tables, accounting for Snowflake's `2024_03` bundle ([#1016](https://github.com/dbt-labs/dbt-snowflake/issues/1016)) +- return to previous naming convention to return to quoting policy ([#1074](https://github.com/dbt-labs/dbt-snowflake/issues/1074)) + + +## dbt-snowflake 1.8.2 - May 23, 2024 + +### Fixes +- Update relation caching to correctly identify dynamic tables, accounting for Snowflake's `2024_03` bundle ([#1016](https://github.com/dbt-labs/dbt-snowflake/issues/1016)) ## dbt-snowflake 1.8.1 - May 15, 2024 diff --git a/dbt/adapters/snowflake/__version__.py b/dbt/adapters/snowflake/__version__.py index ba22724db..eaf9d19e0 100644 --- a/dbt/adapters/snowflake/__version__.py +++ b/dbt/adapters/snowflake/__version__.py @@ -1 +1 @@ -version = "1.8.2" +version = "1.8.3" From bb7f8cbd2052ee8b67683339bab65c5a24038370 Mon Sep 17 00:00:00 2001 From: Matthew McKnight <91097623+McKnight-42@users.noreply.github.com> Date: Fri, 28 Jun 2024 16:00:33 -0500 Subject: [PATCH 3/9] manual backport of 1096 to update secrets_test (#1099) --- tests/functional/adapter/test_python_model.py | 36 +++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/tests/functional/adapter/test_python_model.py b/tests/functional/adapter/test_python_model.py index 6bf0678c7..86ea0a346 100644 --- a/tests/functional/adapter/test_python_model.py +++ b/tests/functional/adapter/test_python_model.py @@ -1,4 +1,5 @@ import pytest +import uuid from dbt.tests.util import run_dbt, write_file from dbt.tests.adapter.python_model.test_python_model import ( BasePythonModelTests, @@ -174,20 +175,24 @@ def test_external_access_integration(self, project): run_dbt(["run"]) -SECRETS_MODE = """ +TEST_RUN_ID = uuid.uuid4().hex +TEST_SECRET = f"test_secret_{TEST_RUN_ID}" +TEST_NETWORK_RULE = f"test_network_rule_{TEST_RUN_ID}" +TEST_EXTERNAL_ACCESS_INTEGRATION = f"test_external_access_integration_{TEST_RUN_ID}" +SECRETS_MODE = f""" import pandas import snowflake.snowpark as snowpark def model(dbt, session: snowpark.Session): dbt.config( materialized="table", - secrets={"secret_variable_name": "test_secret"}, - external_access_integrations=["test_external_access_integration"], + secrets={{"secret_variable_name": "{TEST_SECRET}"}}, + external_access_integrations=["{TEST_EXTERNAL_ACCESS_INTEGRATION}"], ) import _snowflake return session.create_dataframe( pandas.DataFrame( - [{"secret_value": _snowflake.get_generic_secret_string('secret_variable_name')}] + [{{"secret_value": _snowflake.get_generic_secret_string('secret_variable_name')}}] ) ) """ @@ -198,18 +203,29 @@ class TestSecrets: def models(self): return {"secret_python_model.py": SECRETS_MODE} + @pytest.fixture(scope="class") + def profiles_config_update(self): + return {"retry_all": True, "connect_retries": 3} + def test_secrets(self, project): project.run_sql( - "create or replace secret test_secret type = generic_string secret_string='secret value';" + f"create or replace secret {TEST_SECRET} type = generic_string secret_string='secret value';" ) - # The secrets you specify as values must also be specified in the external access integration. - # See https://docs.snowflake.com/en/developer-guide/external-network-access/creating-using-external-network-access#using-the-external-access-integration-in-a-function-or-procedure - project.run_sql( - "create or replace network rule test_network_rule type = host_port mode = egress value_list= ('www.google.com:443');" + f"create or replace network rule {TEST_NETWORK_RULE} type = host_port mode = egress value_list= ('www.google.com:443');" ) + project.run_sql( - "create or replace external access integration test_external_access_integration allowed_network_rules = (test_network_rule) allowed_authentication_secrets = (test_secret) enabled = true;" + f"create or replace external access integration {TEST_EXTERNAL_ACCESS_INTEGRATION} " + f"allowed_network_rules = ({TEST_NETWORK_RULE}) " + f"allowed_authentication_secrets = ({TEST_SECRET}) enabled = true;" ) + run_dbt(["run"]) + + project.run_sql(f"drop secret if exists {TEST_SECRET};") + project.run_sql(f"drop network rule if exists {TEST_NETWORK_RULE};") + project.run_sql( + f"drop external access integration if exists {TEST_EXTERNAL_ACCESS_INTEGRATION};" + ) From c28087789dc683a4e28221b58710183b4f1b3634 Mon Sep 17 00:00:00 2001 From: Matthew McKnight <91097623+McKnight-42@users.noreply.github.com> Date: Mon, 29 Jul 2024 18:29:59 -0500 Subject: [PATCH 4/9] pinning to v1.3.2 allows all tests to pass (#1150) * pinning to v1.3.2 allows all tests to pass * removing ponter to 1.8.latest for dbt-core --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index a2a76eb52..6f4fd46ae 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,6 +1,6 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? -git+https://github.com/dbt-labs/dbt-core.git@1.8.latest#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core git+https://github.com/dbt-labs/dbt-adapters.git git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter From 60c514f38f42db8c80c0fd9e7fe122031cce1985 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 11 Oct 2024 16:17:26 -0700 Subject: [PATCH 5/9] [Bug] Fix scenarios where `--empty` flag pushes `limit 0` into metadata queries (#1100) (#1204) * fix get_columns_in_relation * add render to relation in other metadata queries * add tests for all render instances (cherry picked from commit 13970da86c870dcba79d6c603b0fd35f5ca86a9d) Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .../unreleased/Fixes-20240628-190140.yaml | 7 ++ dbt/include/snowflake/macros/adapters.sql | 16 ++--- tests/functional/adapter/empty/_models.py | 72 +++++++++++++++++++ tests/functional/adapter/empty/test_empty.py | 42 +++++++++++ 4 files changed, 129 insertions(+), 8 deletions(-) create mode 100644 .changes/unreleased/Fixes-20240628-190140.yaml create mode 100644 tests/functional/adapter/empty/_models.py diff --git a/.changes/unreleased/Fixes-20240628-190140.yaml b/.changes/unreleased/Fixes-20240628-190140.yaml new file mode 100644 index 000000000..c58b465fd --- /dev/null +++ b/.changes/unreleased/Fixes-20240628-190140.yaml @@ -0,0 +1,7 @@ +kind: Fixes +body: Fix scenario where using the `--empty` flag causes metadata queries to contain + limit clauses +time: 2024-06-28T19:01:40.558234-04:00 +custom: + Author: mikealfare + Issue: "1033" diff --git a/dbt/include/snowflake/macros/adapters.sql b/dbt/include/snowflake/macros/adapters.sql index 6e7ea8f6c..f0ac8ab35 100644 --- a/dbt/include/snowflake/macros/adapters.sql +++ b/dbt/include/snowflake/macros/adapters.sql @@ -27,14 +27,14 @@ {% macro snowflake__get_columns_in_relation(relation) -%} {%- set sql -%} - describe table {{ relation }} + describe table {{ relation.render() }} {%- endset -%} {%- set result = run_query(sql) -%} {% set maximum = 10000 %} {% if (result | length) >= maximum %} {% set msg %} - Too many columns in relation {{ relation }}! dbt can only get + Too many columns in relation {{ relation.render() }}! dbt can only get information about relations with fewer than {{ maximum }} columns. {% endset %} {% do exceptions.raise_compiler_error(msg) %} @@ -177,7 +177,7 @@ {% macro snowflake__alter_column_type(relation, column_name, new_column_type) -%} {% call statement('alter_column_type') %} - alter table {{ relation }} alter {{ adapter.quote(column_name) }} set data type {{ new_column_type }}; + alter table {{ relation.render() }} alter {{ adapter.quote(column_name) }} set data type {{ new_column_type }}; {% endcall %} {% endmacro %} @@ -187,7 +187,7 @@ {%- else -%} {%- set relation_type = relation.type -%} {%- endif -%} - comment on {{ relation_type }} {{ relation }} IS $${{ relation_comment | replace('$', '[$]') }}$$; + comment on {{ relation_type }} {{ relation.render() }} IS $${{ relation_comment | replace('$', '[$]') }}$$; {% endmacro %} @@ -198,7 +198,7 @@ {% else -%} {% set relation_type = relation.type %} {% endif %} - alter {{ relation_type }} {{ relation }} alter + alter {{ relation_type }} {{ relation.render() }} alter {% for column_name in existing_columns if (column_name in existing_columns) or (column_name|lower in existing_columns) %} {{ get_column_comment_sql(column_name, column_dict) }} {{- ',' if not loop.last else ';' }} {% endfor %} @@ -257,7 +257,7 @@ {% if add_columns %} {% set sql -%} - alter {{ relation_type }} {{ relation }} add column + alter {{ relation_type }} {{ relation.render() }} add column {% for column in add_columns %} {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }} {% endfor %} @@ -270,7 +270,7 @@ {% if remove_columns %} {% set sql -%} - alter {{ relation_type }} {{ relation }} drop column + alter {{ relation_type }} {{ relation.render() }} drop column {% for column in remove_columns %} {{ column.name }}{{ ',' if not loop.last }} {% endfor %} @@ -303,7 +303,7 @@ {% macro snowflake__truncate_relation(relation) -%} {% set truncate_dml %} - truncate table {{ relation }} + truncate table {{ relation.render() }} {% endset %} {% call statement('truncate_relation') -%} {{ snowflake_dml_explicit_transaction(truncate_dml) }} diff --git a/tests/functional/adapter/empty/_models.py b/tests/functional/adapter/empty/_models.py new file mode 100644 index 000000000..698e4f1db --- /dev/null +++ b/tests/functional/adapter/empty/_models.py @@ -0,0 +1,72 @@ +SEED = """ +my_id,my_value +1,a +2,b +3,c +""".strip() + + +SCHEMA = """ +version: 2 + +seeds: + - name: my_seed + description: "This is my_seed" + columns: + - name: id + description: "This is my_seed.my_id" +""" + +CONTROL = """ +select * from {{ ref("my_seed") }} +""" + + +GET_COLUMNS_IN_RELATION = """ +{{ config(materialized="table") }} +{% set columns = adapter.get_columns_in_relation(ref("my_seed")) %} +select * from {{ ref("my_seed") }} +""" + + +ALTER_COLUMN_TYPE = """ +{{ config(materialized="table") }} +{{ alter_column_type(ref("my_seed"), "MY_VALUE", "varchar") }} +select * from {{ ref("my_seed") }} +""" + + +ALTER_RELATION_COMMENT = """ +{{ config( + materialized="table", + persist_docs={"relations": True}, +) }} +select * from {{ ref("my_seed") }} +""" + + +ALTER_COLUMN_COMMENT = """ +{{ config( + materialized="table", + persist_docs={"columns": True}, +) }} +select * from {{ ref("my_seed") }} +""" + + +ALTER_RELATION_ADD_REMOVE_COLUMNS = """ +{{ config(materialized="table") }} +{% set my_seed = adapter.Relation.create(this.database, this.schema, "my_seed", "table") %} +{% set my_column = api.Column("my_column", "varchar") %} +{% do alter_relation_add_remove_columns(my_seed, [my_column], none) %} +{% do alter_relation_add_remove_columns(my_seed, none, [my_column]) %} +select * from {{ ref("my_seed") }} +""" + + +TRUNCATE_RELATION = """ +{{ config(materialized="table") }} +{% set my_seed = adapter.Relation.create(this.database, this.schema, "my_seed", "table") %} +{{ truncate_relation(my_seed) }} +select * from {{ ref("my_seed") }} +""" diff --git a/tests/functional/adapter/empty/test_empty.py b/tests/functional/adapter/empty/test_empty.py index 401179b71..fe07fc081 100644 --- a/tests/functional/adapter/empty/test_empty.py +++ b/tests/functional/adapter/empty/test_empty.py @@ -1,4 +1,8 @@ from dbt.tests.adapter.empty.test_empty import BaseTestEmpty, BaseTestEmptyInlineSourceRef +from dbt.tests.util import run_dbt +import pytest + +from tests.functional.adapter.empty import _models class TestSnowflakeEmpty(BaseTestEmpty): @@ -7,3 +11,41 @@ class TestSnowflakeEmpty(BaseTestEmpty): class TestSnowflakeEmptyInlineSourceRef(BaseTestEmptyInlineSourceRef): pass + + +class TestMetadataWithEmptyFlag: + @pytest.fixture(scope="class") + def seeds(self): + return {"my_seed.csv": _models.SEED} + + @pytest.fixture(scope="class") + def models(self): + return { + "schema.yml": _models.SCHEMA, + "control.sql": _models.CONTROL, + "get_columns_in_relation.sql": _models.GET_COLUMNS_IN_RELATION, + "alter_column_type.sql": _models.ALTER_COLUMN_TYPE, + "alter_relation_comment.sql": _models.ALTER_RELATION_COMMENT, + "alter_column_comment.sql": _models.ALTER_COLUMN_COMMENT, + "alter_relation_add_remove_columns.sql": _models.ALTER_RELATION_ADD_REMOVE_COLUMNS, + "truncate_relation.sql": _models.TRUNCATE_RELATION, + } + + @pytest.fixture(scope="class", autouse=True) + def setup(self, project): + run_dbt(["seed"]) + + @pytest.mark.parametrize( + "model", + [ + "control", + "get_columns_in_relation", + "alter_column_type", + "alter_relation_comment", + "alter_column_comment", + "alter_relation_add_remove_columns", + "truncate_relation", + ], + ) + def test_run(self, project, model): + run_dbt(["run", "--empty", "--select", model]) From 878192d467edc9ede1508308c77d8bef5783b406 Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Thu, 17 Oct 2024 21:16:02 +0000 Subject: [PATCH 6/9] Bumping version to 1.8.4 and generate changelog --- .bumpversion.cfg | 2 +- .changes/1.8.4.md | 5 +++++ .changes/unreleased/Fixes-20240628-190140.yaml | 7 ------- CHANGELOG.md | 10 ++++++++-- dbt/adapters/snowflake/__version__.py | 2 +- 5 files changed, 15 insertions(+), 11 deletions(-) create mode 100644 .changes/1.8.4.md delete mode 100644 .changes/unreleased/Fixes-20240628-190140.yaml diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 22b3437af..ccf54846a 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.8.3 +current_version = 1.8.4 parse = (?P[\d]+) # major version number \.(?P[\d]+) # minor version number \.(?P[\d]+) # patch version number diff --git a/.changes/1.8.4.md b/.changes/1.8.4.md new file mode 100644 index 000000000..f3482d24d --- /dev/null +++ b/.changes/1.8.4.md @@ -0,0 +1,5 @@ +## dbt-snowflake 1.8.4 - October 17, 2024 + +### Fixes + +- Fix scenario where using the `--empty` flag causes metadata queries to contain limit clauses ([#1033](https://github.com/dbt-labs/dbt-snowflake/issues/1033)) diff --git a/.changes/unreleased/Fixes-20240628-190140.yaml b/.changes/unreleased/Fixes-20240628-190140.yaml deleted file mode 100644 index c58b465fd..000000000 --- a/.changes/unreleased/Fixes-20240628-190140.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: Fixes -body: Fix scenario where using the `--empty` flag causes metadata queries to contain - limit clauses -time: 2024-06-28T19:01:40.558234-04:00 -custom: - Author: mikealfare - Issue: "1033" diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e026c5c0..fcaabbd7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,13 +5,19 @@ - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version. - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-snowflake/blob/main/CONTRIBUTING.md#adding-changelog-entry) -## dbt-snowflake 1.8.3 - June 12, 2024 +## dbt-snowflake 1.8.4 - October 17, 2024 ### Fixes -- return to previous naming convention to return to quoting policy ([#1074](https://github.com/dbt-labs/dbt-snowflake/issues/1074)) +- Fix scenario where using the `--empty` flag causes metadata queries to contain limit clauses ([#1033](https://github.com/dbt-labs/dbt-snowflake/issues/1033)) + + +## dbt-snowflake 1.8.3 - June 12, 2024 + +### Fixes +- return to previous naming convention to return to quoting policy ([#1074](https://github.com/dbt-labs/dbt-snowflake/issues/1074)) ## dbt-snowflake 1.8.2 - May 23, 2024 diff --git a/dbt/adapters/snowflake/__version__.py b/dbt/adapters/snowflake/__version__.py index eaf9d19e0..f2493fc98 100644 --- a/dbt/adapters/snowflake/__version__.py +++ b/dbt/adapters/snowflake/__version__.py @@ -1 +1 @@ -version = "1.8.3" +version = "1.8.4" From 7cc4730ca23a0d2ab12e08cae71532831bf713ab Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Mon, 21 Oct 2024 17:53:48 -0400 Subject: [PATCH 7/9] [Backport 1.8.latest] Drop support for Python 3.8 (#1216) * backport #1211 --- .../unreleased/Breaking Changes-20241016-183143.yaml | 6 ++++++ .github/scripts/integration-test-matrix.js | 4 ++-- .github/workflows/main.yml | 8 ++++---- .pre-commit-config.yaml | 4 ++-- CONTRIBUTING.md | 2 +- Makefile | 12 ++++++------ setup.py | 8 +++----- tox.ini | 6 +++--- 8 files changed, 27 insertions(+), 23 deletions(-) create mode 100644 .changes/unreleased/Breaking Changes-20241016-183143.yaml diff --git a/.changes/unreleased/Breaking Changes-20241016-183143.yaml b/.changes/unreleased/Breaking Changes-20241016-183143.yaml new file mode 100644 index 000000000..26cc4b6de --- /dev/null +++ b/.changes/unreleased/Breaking Changes-20241016-183143.yaml @@ -0,0 +1,6 @@ +kind: Breaking Changes +body: Drop support for Python 3.8 +time: 2024-10-16T18:31:43.4167-04:00 +custom: + Author: mikealfare + Issue: "1211" diff --git a/.github/scripts/integration-test-matrix.js b/.github/scripts/integration-test-matrix.js index ceb01cd2e..60afa2b61 100644 --- a/.github/scripts/integration-test-matrix.js +++ b/.github/scripts/integration-test-matrix.js @@ -1,6 +1,6 @@ module.exports = ({ context }) => { - const defaultPythonVersion = "3.8"; - const supportedPythonVersions = ["3.8", "3.9", "3.10", "3.11"]; + const defaultPythonVersion = "3.9"; + const supportedPythonVersions = ["3.9", "3.10", "3.11"]; const supportedAdapters = ["snowflake"]; // if PR, generate matrix based on files changed and PR labels diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 25a3ef10e..39286a2c9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -50,7 +50,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.9' - name: Install python dependencies run: | @@ -72,7 +72,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] + python-version: ['3.9', '3.10', '3.11'] env: TOXENV: "unit" @@ -127,7 +127,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.9' - name: Install python dependencies run: | @@ -174,7 +174,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-12, windows-latest] - python-version: ['3.8', '3.9', '3.10', '3.11'] + python-version: ['3.9', '3.10', '3.11'] steps: - name: Set up Python ${{ matrix.python-version }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b89d0df27..b620d4ca3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,14 +25,14 @@ repos: additional_dependencies: ['click~=8.1'] args: - "--line-length=99" - - "--target-version=py38" + - "--target-version=py39" - id: black alias: black-check stages: [manual] additional_dependencies: ['click~=8.1'] args: - "--line-length=99" - - "--target-version=py38" + - "--target-version=py39" - "--check" - "--diff" - repo: https://github.com/pycqa/flake8 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 45e0054c5..5b68aa03a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -79,7 +79,7 @@ WARNING: The parameters in your `test.env` file must link to a valid Snowflake a There are a few methods for running tests locally. #### `tox` -`tox` automatically runs unit tests against several Python versions using its own virtualenvs. Run `tox -p` to run unit tests for Python 3.8, Python 3.9, Python 3.10, and `flake8` in parallel. Run `tox -e py38` to invoke tests on Python version 3.8 only (use py38, py39, or py310). Tox recipes are found in `tox.ini`. +`tox` automatically runs unit tests against several Python versions using its own virtualenvs. Run `tox -p` to run unit tests for Python 3.9 and Python 3.10, and `flake8` in parallel. Run `tox -e py39` to invoke tests on Python version 3.9 only (use py39 or py310). Tox recipes are found in `tox.ini`. #### `pytest` You may run a specific test or group of tests using `pytest` directly. Activate a Python virtualenv active with dev dependencies installed. Then, run tests like so: diff --git a/Makefile b/Makefile index fc1511a74..8d2950235 100644 --- a/Makefile +++ b/Makefile @@ -38,22 +38,22 @@ linecheck: ## Checks for all Python lines 100 characters or more find dbt -type f -name "*.py" -exec grep -I -r -n '.\{100\}' {} \; .PHONY: unit -unit: ## Runs unit tests with py38. +unit: ## Runs unit tests with py39. @\ - tox -e py38 + tox -e py39 .PHONY: test -test: ## Runs unit tests with py38 and code checks against staged changes. +test: ## Runs unit tests with py39 and code checks against staged changes. @\ - tox -p -e py38; \ + tox -p -e py39; \ pre-commit run black-check --hook-stage manual | grep -v "INFO"; \ pre-commit run flake8-check --hook-stage manual | grep -v "INFO"; \ pre-commit run mypy-check --hook-stage manual | grep -v "INFO" .PHONY: integration -integration: ## Runs snowflake integration tests with py38. +integration: ## Runs snowflake integration tests with py39. @\ - tox -e py38-snowflake -- + tox -e py39-snowflake -- .PHONY: clean @echo "cleaning repo" diff --git a/setup.py b/setup.py index f8ff363ed..f28af5d76 100644 --- a/setup.py +++ b/setup.py @@ -5,9 +5,9 @@ import sys # require python 3.8 or newer -if sys.version_info < (3, 8): +if sys.version_info < (3, 9): print("Error: dbt does not support this version of Python.") - print("Please upgrade to Python 3.8 or higher.") + print("Please upgrade to Python 3.9 or higher.") sys.exit(1) @@ -72,11 +72,9 @@ def _plugin_version() -> str: "Operating System :: Microsoft :: Windows", "Operating System :: MacOS :: MacOS X", "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", ], - python_requires=">=3.8", + python_requires=">=3.9", ) diff --git a/tox.ini b/tox.ini index 4697044da..fd317f688 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,8 @@ [tox] skipsdist = True -envlist = py38,py39,py310,py311 +envlist = py39,py310,py311 -[testenv:{unit,py38,py39,py310,py311,py}] +[testenv:{unit,py39,py310,py311,py}] description = unit testing skip_install = true passenv = @@ -13,7 +13,7 @@ deps = -rdev-requirements.txt -e. -[testenv:{integration,py38,py39,py310,py311,py}-{snowflake}] +[testenv:{integration,py39,py310,py311,py}-{snowflake}] description = adapter plugin integration testing skip_install = true passenv = From 243ea4f05762b4ba1d0582b640b11cf83004d6a7 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Fri, 8 Nov 2024 14:05:33 -0500 Subject: [PATCH 8/9] update mocks to account for new query_id in the adapter response (#1238) --- tests/unit/test_snowflake_adapter.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unit/test_snowflake_adapter.py b/tests/unit/test_snowflake_adapter.py index 2666e7557..7a75983f3 100644 --- a/tests/unit/test_snowflake_adapter.py +++ b/tests/unit/test_snowflake_adapter.py @@ -59,6 +59,9 @@ def setUp(self): self.handle = mock.MagicMock(spec=snowflake_connector.SnowflakeConnection) self.cursor = self.handle.cursor.return_value + # query_id needs to be a string for the adapter response protobuf event + self.sfqid = mock.patch.object(self.cursor, "sfqid", new_callable=lambda: "42") + self.sfqid.start() self.mock_execute = self.cursor.execute self.patcher = mock.patch("dbt.adapters.snowflake.connections.snowflake.connector.connect") self.snowflake = self.patcher.start() @@ -99,6 +102,7 @@ def tearDown(self): self.adapter.cleanup_connections() self.qh_patch.stop() self.patcher.stop() + self.sfqid.stop() self.load_state_check.stop() def test_quoting_on_drop_schema(self): From a6e665bd8d178e3606276246c8a019e1dc991ca0 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Fri, 8 Nov 2024 16:21:57 -0500 Subject: [PATCH 9/9] move github runner from macos-12 to macos-14 (#1230) (#1239) * move github runner from macos-12 to macos-14 (cherry picked from commit a16654bed4d67d1a922bf87e579a285c35bd55ac) Co-authored-by: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> --- .github/scripts/integration-test-matrix.js | 4 ++-- .github/workflows/main.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/integration-test-matrix.js b/.github/scripts/integration-test-matrix.js index 60afa2b61..b72a955f9 100644 --- a/.github/scripts/integration-test-matrix.js +++ b/.github/scripts/integration-test-matrix.js @@ -44,7 +44,7 @@ module.exports = ({ context }) => { if (labels.includes("test macos") || testAllLabel) { include.push({ - os: "macos-12", + os: "macos-14", adapter, "python-version": pythonVersion, }); @@ -78,7 +78,7 @@ module.exports = ({ context }) => { // additionally include runs for all adapters, on macos and windows, // but only for the default python version for (const adapter of supportedAdapters) { - for (const operatingSystem of ["windows-latest", "macos-12"]) { + for (const operatingSystem of ["windows-latest", "macos-14"]) { include.push({ os: operatingSystem, adapter: adapter, diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 39286a2c9..01d8f298b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -173,7 +173,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-12, windows-latest] + os: [ubuntu-latest, macos-14, windows-latest] python-version: ['3.9', '3.10', '3.11'] steps: