diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 37e27dcb2..6c726eadc 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -3,7 +3,10 @@ import agate from dbt.adapters.sql import SQLAdapter -from dbt.adapters.sql.impl import LIST_SCHEMAS_MACRO_NAME +from dbt.adapters.sql.impl import ( + LIST_SCHEMAS_MACRO_NAME, + LIST_RELATIONS_MACRO_NAME, +) from dbt.adapters.snowflake import SnowflakeConnectionManager from dbt.adapters.snowflake import SnowflakeRelation from dbt.adapters.snowflake import SnowflakeColumn @@ -99,3 +102,43 @@ def list_schemas(self, database: str) -> List[str]: # want is 'name' return [row['name'] for row in results] + + def list_relations_without_caching( + self, information_schema, schema + ) -> List[SnowflakeRelation]: + kwargs = {'information_schema': information_schema, 'schema': schema} + try: + results = self.execute_macro( + LIST_RELATIONS_MACRO_NAME, + kwargs=kwargs + ) + except DatabaseException as exc: + # if the schema doesn't exist, we just want to return. + # Alternatively, we could query the list of schemas before we start + # and skip listing the missing ones, which sounds expensive. + if 'Object does not exist' in str(exc): + return [] + raise + + relations = [] + quote_policy = { + 'database': True, + 'schema': True, + 'identifier': True + } + + columns = ['database_name', 'schema_name', 'name', 'kind'] + for _database, _schema, _identifier, _type in results.select(columns): + try: + _type = self.Relation.get_relation_type(_type.lower()) + except ValueError: + _type = self.Relation.External + relations.append(self.Relation.create( + database=_database, + schema=_schema, + identifier=_identifier, + quote_policy=quote_policy, + type=_type + )) + + return relations diff --git a/dbt/include/snowflake/macros/adapters.sql b/dbt/include/snowflake/macros/adapters.sql index 9d051c4e3..38296ec95 100644 --- a/dbt/include/snowflake/macros/adapters.sql +++ b/dbt/include/snowflake/macros/adapters.sql @@ -99,22 +99,22 @@ {% macro snowflake__list_relations_without_caching(information_schema, schema) %} - {% call statement('list_relations_without_caching', fetch_result=True) -%} - select - table_catalog as database, - table_name as name, - table_schema as schema, - case when table_type = 'BASE TABLE' then 'table' - when table_type = 'VIEW' then 'view' - when table_type = 'MATERIALIZED VIEW' then 'materializedview' - when table_type = 'EXTERNAL TABLE' then 'external' - else table_type - end as table_type - from {{ information_schema }}.tables - where table_schema ilike '{{ schema }}' - and table_catalog ilike '{{ information_schema.database.lower() }}' - {% endcall %} - {{ return(load_result('list_relations_without_caching').table) }} + {%- set db_name = adapter.quote_as_configured(information_schema.database, 'database') -%} + {%- set schema_name = adapter.quote_as_configured(schema, 'schema') -%} + {%- set sql -%} + show terse objects in {{ db_name }}.{{ schema_name }} + {%- endset -%} + + {%- set result = run_query(sql) -%} + {% set maximum = 10000 %} + {% if (result | length) >= maximum %} + {% set msg %} + Too many schemas in schema {{ database }}.{{ schema }}! dbt can only get + information about schemas with fewer than {{ maximum }} objects. + {% endset %} + {% do exceptions.raise_compiler_error(msg) %} + {% endif %} + {%- do return(result) -%} {% endmacro %}