Merge pull request #23 from SpotOnInc/metadata-match-by-column-name

add in metadata for match_by_column_name

dbt_project.yml

@@ -1,7 +1,9 @@
 name: 'dbt_object_mgmt'
-version: '0.2.7'
+version: '0.2.10'
 config-version: 2
 
+profile: 'dbt_object_mgmt'
+
 target-path: "target"
 macro-paths: ["macros"]
 log-path: "logs"
@@ -14,3 +16,4 @@ vars:
   snowflake_user_file: snowflake/users/users.yml
   snowflake_network_policy_file: snowflake/policy/network_policies.yml
   snowflake_admin: securityadmin
+  dry_run: true

macros/create_pipe.sql

@@ -28,6 +28,15 @@
   {{ format_type_options.update(pipe.extra_format_options) }}
 {% endif %}
 
+{% set metadata_columns = {
+      'file_name': 'metadata$filename',
+      'file_id': 'metadata$file_content_key',
+      'row_number': 'metadata$file_row_number',
+      'last_modified_time': 'metadata$file_last_modified',
+      'load_time': 'metadata$start_scan_time',
+  }
+%}
+
 {% set copy_statement -%}
 copy into {{ schema_name }}.{{ table_name }} from
 {% if pipe.match_by_column_name %}
@@ -39,6 +48,15 @@ copy into {{ schema_name }}.{{ table_name }} from
   -#}
   @{{ schema_name }}.{{ table_name }}_stage
   {{ "match_by_column_name = " ~ pipe.match_by_column_name }}
+
+  {% if pipe.match_by_column_name -%}
+    include_metadata = (
+      {% for key, value in metadata_columns.items() %}
+        {{- key }} = {{ value }}{{ ', ' if not loop.last }}
+      {% endfor %}
+    )
+  {%- endif %}
+
 {% else %} (
     select
       {%- if file_type == 'JSON' %}
@@ -48,11 +66,7 @@ copy into {{ schema_name }}.{{ table_name }} from
       {{ ', ' if not loop.first }}${{ loop.index }}
       {%- endfor %}
       {%- endif %}
-      , metadata$filename
-      , md5(metadata$filename)
-      , metadata$file_row_number
-      , metadata$file_last_modified
-      , metadata$start_scan_time
+      , {{ metadata_columns.values() | join(', ') -}}
     from
       @{{ schema_name }}.{{ table_name }}_stage
   )
@@ -63,8 +77,8 @@ copy into {{ schema_name }}.{{ table_name }} from
     type = '{{ file_type }}'
     {% for key, value in format_type_options.items() %}
       {{- key }} = {{ value }}
-    {% endfor -%}
-    )
+    {% endfor %}
+  )
 {% endset %}
 
 {%- set sql -%}

snowflake/snowpipe/s3_pipe_jaffle_shop_customers_match_column_name.yml

@@ -0,0 +1,14 @@
+integration_name: jaffle_shop_integration
+schema_name: raw_jaffle_shop
+table_name: jaffle_shop_customers
+s3_url: 's3://jaffle_shop/customers/'
+file_type: 'CSV'
+match_by_column_name: case_insensitive
+extra_format_options:
+  parse_header: true
+  skip_header: 0
+  error_on_column_count_mismatch: false
+columns:
+  id: text
+  first_name: text
+  last_name: text