diff --git a/dbt_project.yml b/dbt_project.yml index e2774d6..d604f83 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,7 +1,9 @@ name: 'dbt_object_mgmt' -version: '0.2.7' +version: '0.2.10' config-version: 2 +profile: 'dbt_object_mgmt' + target-path: "target" macro-paths: ["macros"] log-path: "logs" @@ -14,3 +16,4 @@ vars: snowflake_user_file: snowflake/users/users.yml snowflake_network_policy_file: snowflake/policy/network_policies.yml snowflake_admin: securityadmin + dry_run: true diff --git a/macros/create_pipe.sql b/macros/create_pipe.sql index 9f6441a..54ee1b9 100644 --- a/macros/create_pipe.sql +++ b/macros/create_pipe.sql @@ -28,6 +28,15 @@ {{ format_type_options.update(pipe.extra_format_options) }} {% endif %} +{% set metadata_columns = { + 'file_name': 'metadata$filename', + 'file_id': 'metadata$file_content_key', + 'row_number': 'metadata$file_row_number', + 'last_modified_time': 'metadata$file_last_modified', + 'load_time': 'metadata$start_scan_time', + } +%} + {% set copy_statement -%} copy into {{ schema_name }}.{{ table_name }} from {% if pipe.match_by_column_name %} @@ -39,6 +48,15 @@ copy into {{ schema_name }}.{{ table_name }} from -#} @{{ schema_name }}.{{ table_name }}_stage {{ "match_by_column_name = " ~ pipe.match_by_column_name }} + + {% if pipe.match_by_column_name -%} + include_metadata = ( + {% for key, value in metadata_columns.items() %} + {{- key }} = {{ value }}{{ ', ' if not loop.last }} + {% endfor %} + ) + {%- endif %} + {% else %} ( select {%- if file_type == 'JSON' %} @@ -48,11 +66,7 @@ copy into {{ schema_name }}.{{ table_name }} from {{ ', ' if not loop.first }}${{ loop.index }} {%- endfor %} {%- endif %} - , metadata$filename - , md5(metadata$filename) - , metadata$file_row_number - , metadata$file_last_modified - , metadata$start_scan_time + , {{ metadata_columns.values() | join(', ') -}} from @{{ schema_name }}.{{ table_name }}_stage ) @@ -63,8 +77,8 @@ copy into {{ schema_name }}.{{ table_name }} from type = '{{ file_type }}' {% for key, value in format_type_options.items() %} {{- key }} = {{ value }} - {% endfor -%} - ) + {% endfor %} + ) {% endset %} {%- set sql -%} diff --git a/snowflake/snowpipe/s3_pipe_jaffle_shop_customers_match_column_name.yml b/snowflake/snowpipe/s3_pipe_jaffle_shop_customers_match_column_name.yml new file mode 100644 index 0000000..c00ea48 --- /dev/null +++ b/snowflake/snowpipe/s3_pipe_jaffle_shop_customers_match_column_name.yml @@ -0,0 +1,14 @@ +integration_name: jaffle_shop_integration +schema_name: raw_jaffle_shop +table_name: jaffle_shop_customers +s3_url: 's3://jaffle_shop/customers/' +file_type: 'CSV' +match_by_column_name: case_insensitive +extra_format_options: + parse_header: true + skip_header: 0 + error_on_column_count_mismatch: false +columns: + id: text + first_name: text + last_name: text