Skip to content

Commit

Permalink
change list_relations_without_caching to use "show objects in schema"
Browse files Browse the repository at this point in the history
  • Loading branch information
Jacob Beck committed Apr 14, 2020
1 parent 7e73166 commit 6c43ccf
Show file tree
Hide file tree
Showing 2 changed files with 60 additions and 17 deletions.
45 changes: 44 additions & 1 deletion dbt/adapters/snowflake/impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,10 @@
import agate

from dbt.adapters.sql import SQLAdapter
from dbt.adapters.sql.impl import LIST_SCHEMAS_MACRO_NAME
from dbt.adapters.sql.impl import (
LIST_SCHEMAS_MACRO_NAME,
LIST_RELATIONS_MACRO_NAME,
)
from dbt.adapters.snowflake import SnowflakeConnectionManager
from dbt.adapters.snowflake import SnowflakeRelation
from dbt.adapters.snowflake import SnowflakeColumn
Expand Down Expand Up @@ -99,3 +102,43 @@ def list_schemas(self, database: str) -> List[str]:
# want is 'name'

return [row['name'] for row in results]

def list_relations_without_caching(
self, information_schema, schema
) -> List[SnowflakeRelation]:
kwargs = {'information_schema': information_schema, 'schema': schema}
try:
results = self.execute_macro(
LIST_RELATIONS_MACRO_NAME,
kwargs=kwargs
)
except DatabaseException as exc:
# if the schema doesn't exist, we just want to return.
# Alternatively, we could query the list of schemas before we start
# and skip listing the missing ones, which sounds expensive.
if 'Object does not exist' in str(exc):
return []
raise

relations = []
quote_policy = {
'database': True,
'schema': True,
'identifier': True
}

columns = ['database_name', 'schema_name', 'name', 'kind']
for _database, _schema, _identifier, _type in results.select(columns):
try:
_type = self.Relation.get_relation_type(_type.lower())
except ValueError:
_type = self.Relation.External
relations.append(self.Relation.create(
database=_database,
schema=_schema,
identifier=_identifier,
quote_policy=quote_policy,
type=_type
))

return relations
32 changes: 16 additions & 16 deletions dbt/include/snowflake/macros/adapters.sql
Original file line number Diff line number Diff line change
Expand Up @@ -99,22 +99,22 @@


{% macro snowflake__list_relations_without_caching(information_schema, schema) %}
{% call statement('list_relations_without_caching', fetch_result=True) -%}
select
table_catalog as database,
table_name as name,
table_schema as schema,
case when table_type = 'BASE TABLE' then 'table'
when table_type = 'VIEW' then 'view'
when table_type = 'MATERIALIZED VIEW' then 'materializedview'
when table_type = 'EXTERNAL TABLE' then 'external'
else table_type
end as table_type
from {{ information_schema }}.tables
where table_schema ilike '{{ schema }}'
and table_catalog ilike '{{ information_schema.database.lower() }}'
{% endcall %}
{{ return(load_result('list_relations_without_caching').table) }}
{%- set db_name = adapter.quote_as_configured(information_schema.database, 'database') -%}
{%- set schema_name = adapter.quote_as_configured(schema, 'schema') -%}
{%- set sql -%}
show terse objects in {{ db_name }}.{{ schema_name }}
{%- endset -%}

{%- set result = run_query(sql) -%}
{% set maximum = 10000 %}
{% if (result | length) >= maximum %}
{% set msg %}
Too many schemas in schema {{ database }}.{{ schema }}! dbt can only get
information about schemas with fewer than {{ maximum }} objects.
{% endset %}
{% do exceptions.raise_compiler_error(msg) %}
{% endif %}
{%- do return(result) -%}
{% endmacro %}


Expand Down

0 comments on commit 6c43ccf

Please sign in to comment.