testing Matrix options for scenarios

Snowflake-Labs · May 6, 2024 · 8655a50 · 8655a50
1 parent 151ea37
commit 8655a50
Show file tree

Hide file tree

Showing 22 changed files with 114 additions and 40 deletions.
diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml
@@ -27,6 +27,8 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
+      matrix:
+        scenario-name: [ 'basics_demo', 'citibike_demo', 'citibike_demo_jinja']
 
     steps:
       - uses: actions/checkout@v4
@@ -61,9 +63,9 @@ jobs:
       #    echo "Tested SnowCLI connection successfully!"
       #    echo "Set default connection"
       #    snow --config-file config.toml connection set-default "myconnection"
-      - name: Test Basic Schemachange
+      - name: Test Schemachange - ${{ matrix.scenario-name }}
         env:
-          SCENARIO_NAME: basics_demo
+          SCENARIO_NAME: ${{ matrix.scenario-name }}
         run: |
           export MY_TARGET_SCHEMA="${SCENARIO_NAME}_${{ env.SCHEMACHANGE_BUILD_ID }}"
           echo "::group::Setting up ${MY_TARGET_SCHEMA}"

diff --git a/demo/citibike/schemachange-config.yml b/demo/citibike/schemachange-config.yml
diff --git a/demo/citibike/A__checks.sql → demo/citibike_demo/A__checks.sql b/demo/citibike/A__checks.sql → demo/citibike_demo/A__checks.sql
diff --git a/demo/citibike/R__checks.sql → demo/citibike_demo/R__checks.sql b/demo/citibike/R__checks.sql → demo/citibike_demo/R__checks.sql
diff --git a/...tibike/V1.1__initial_database_objects.sql → ...e_demo/V1.1__initial_database_objects.sql b/...tibike/V1.1__initial_database_objects.sql → ...e_demo/V1.1__initial_database_objects.sql
@@ -1,7 +1,6 @@
 -- Use Demo Database if it doesn't exist
-USE DATABASE SCHEMACHANGE_DEMO;
--- Set the database and schema context
-USE SCHEMA SCHEMACHANGE_DEMO.CITIBIKE_DEMO;
+use database {{ database_name }};
+use schema {{ schema_name }};
 
 -- Create the file formats
 CREATE OR REPLACE FILE FORMAT CSV_NO_HEADER

diff --git a/demo/citibike/V1.2__load_tables_from_s3.sql → ...tibike_demo/V1.2__load_tables_from_s3.sql b/demo/citibike/V1.2__load_tables_from_s3.sql → ...tibike_demo/V1.2__load_tables_from_s3.sql
@@ -1,5 +1,6 @@
 -- Set the database and schema context
-USE SCHEMA SCHEMACHANGE_DEMO.CITIBIKE_DEMO;
+use database {{ database_name }};
+use schema {{ schema_name }};
 
 -- Load the trips data
 -- Trips data bucket content has been updated. 

diff --git a/demo/citibike_demo/schemachange-config.yml b/demo/citibike_demo/schemachange-config.yml
@@ -0,0 +1,17 @@
+config-version: 1
+
+root-folder: ./demo/{{ env_var('SCENARIO_NAME')}}
+
+snowflake-user: {{ env_var('SNOWFLAKE_USER')}}
+snowflake-account: {{ env_var('SNOWFLAKE_ACCOUNT')}}
+snowflake-role: {{ env_var('SNOWFLAKE_ROLE')}}
+snowflake-warehouse: {{ env_var('SNOWFLAKE_WAREHOUSE')}}
+snowflake-database: {{ env_var('SNOWFLAKE_DATABASE')}}
+snowflake-schema: {{ env_var('MY_TARGET_SCHEMA')}}
+change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.{{ env_var('MY_TARGET_SCHEMA')}}.CHANGE_HISTORY"
+create-change-history-table: true
+
+vars:
+  database_name: {{env_var('SNOWFLAKE_DATABASE')}}
+  schema_name: {{env_var('MY_TARGET_SCHEMA')}}
+
diff --git a/demo/citibike_jinja/modules/create_stage.j2 → ...tibike_demo_jinja/modules/create_stage.j2 b/demo/citibike_jinja/modules/create_stage.j2 → ...tibike_demo_jinja/modules/create_stage.j2
diff --git a/demo/citibike_demo_jinja/schemachange-config.yml b/demo/citibike_demo_jinja/schemachange-config.yml
@@ -0,0 +1,25 @@
+config-version: 1
+
+root-folder: ./demo/{{ env_var('SCENARIO_NAME')}}/scripts
+modules-folder: ./demo/{{ env_var('SCENARIO_NAME')}}/modules
+
+snowflake-user: {{ env_var('SNOWFLAKE_USER')}}
+snowflake-account: {{ env_var('SNOWFLAKE_ACCOUNT')}}
+snowflake-role: {{ env_var('SNOWFLAKE_ROLE')}}
+snowflake-warehouse: {{ env_var('SNOWFLAKE_WAREHOUSE')}}
+snowflake-database: {{ env_var('SNOWFLAKE_DATABASE')}}
+snowflake-schema: {{ env_var('MY_TARGET_SCHEMA')}}
+change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.{{ env_var('MY_TARGET_SCHEMA')}}.CHANGE_HISTORY"
+create-change-history-table: true
+
+vars:
+  database_name: {{env_var('SNOWFLAKE_DATABASE')}}
+  schema_name: {{env_var('MY_TARGET_SCHEMA')}}
+  secrets:
+    # not a good example of secrets, just here to demo the secret filtering
+    trips_s3_bucket: s3://snowflake-workshop-lab/citibike-trips
+    weather_s3_bucket: s3://snowflake-workshop-lab/weather-nyc
+
+
+
+
diff --git a/...cripts/V1.1__initial_database_objects.sql → ...cripts/V1.1__initial_database_objects.sql b/...cripts/V1.1__initial_database_objects.sql → ...cripts/V1.1__initial_database_objects.sql
diff --git a/...nja/scripts/V1.2__load_tables_from_s3.sql → ...nja/scripts/V1.2__load_tables_from_s3.sql b/...nja/scripts/V1.2__load_tables_from_s3.sql → ...nja/scripts/V1.2__load_tables_from_s3.sql
diff --git a/demo/citibike_jinja/schemachange-config.yml b/demo/citibike_jinja/schemachange-config.yml
diff --git a/demo/setup/basics_demo/A__basics_demo.sql → ...etup/basics_demo/A__setup_basics_demo.sql b/demo/setup/basics_demo/A__basics_demo.sql → ...etup/basics_demo/A__setup_basics_demo.sql
diff --git a/...tup/citibike_demo/setup_citibike_demo.sql → .../citibike_demo/A__setup_citibike_demo.sql b/...tup/citibike_demo/setup_citibike_demo.sql → .../citibike_demo/A__setup_citibike_demo.sql
diff --git a/demo/setup/citibike_demo/schemachange-config.yml b/demo/setup/citibike_demo/schemachange-config.yml
@@ -0,0 +1,16 @@
+config-version: 1
+
+root-folder: ./demo/setup/{{ env_var('SCENARIO_NAME')}}
+
+snowflake-user: {{ env_var('SNOWFLAKE_USER')}}
+snowflake-account: {{ env_var('SNOWFLAKE_ACCOUNT')}}
+snowflake-role: {{ env_var('SNOWFLAKE_ROLE')}}
+snowflake-warehouse: {{ env_var('SNOWFLAKE_WAREHOUSE')}}
+snowflake-database: {{ env_var('SNOWFLAKE_DATABASE')}}
+# tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment.
+change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY"
+create-change-history-table: true
+
+vars:
+  database_name: {{env_var('SNOWFLAKE_DATABASE')}}
+  schema_name: {{env_var('MY_TARGET_SCHEMA')}}
diff --git a/..._demo_jinja/setup_citibike_demo_jinja.sql → ...mo_jinja/A__setup_citibike_demo_jinja.sql b/..._demo_jinja/setup_citibike_demo_jinja.sql → ...mo_jinja/A__setup_citibike_demo_jinja.sql
diff --git a/demo/setup/citibike_demo_jinja/schemachange-config.yml b/demo/setup/citibike_demo_jinja/schemachange-config.yml
@@ -0,0 +1,16 @@
+config-version: 1
+
+root-folder: ./demo/setup/{{ env_var('SCENARIO_NAME')}}
+
+snowflake-user: {{ env_var('SNOWFLAKE_USER')}}
+snowflake-account: {{ env_var('SNOWFLAKE_ACCOUNT')}}
+snowflake-role: {{ env_var('SNOWFLAKE_ROLE')}}
+snowflake-warehouse: {{ env_var('SNOWFLAKE_WAREHOUSE')}}
+snowflake-database: {{ env_var('SNOWFLAKE_DATABASE')}}
+# tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment.
+change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY"
+create-change-history-table: true
+
+vars:
+  database_name: {{env_var('SNOWFLAKE_DATABASE')}}
+  schema_name: {{env_var('MY_TARGET_SCHEMA')}}
diff --git a/demo/teardown/basics_demo/A__basics_demo.sql → ...n/basics_demo/A__teardown_basics_demo.sql b/demo/teardown/basics_demo/A__basics_demo.sql → ...n/basics_demo/A__teardown_basics_demo.sql
diff --git a/.../citibike_demo/teardown_citibike_demo.sql → ...tibike_demo/A__teardown_citibike_demo.sql b/.../citibike_demo/teardown_citibike_demo.sql → ...tibike_demo/A__teardown_citibike_demo.sql
diff --git a/demo/teardown/citibike_demo/schemachange-config.yml b/demo/teardown/citibike_demo/schemachange-config.yml
@@ -0,0 +1,16 @@
+config-version: 1
+
+root-folder: ./demo/teardown/{{ env_var('SCENARIO_NAME')}}
+
+snowflake-user: {{ env_var('SNOWFLAKE_USER')}}
+snowflake-account: {{ env_var('SNOWFLAKE_ACCOUNT')}}
+snowflake-role: {{ env_var('SNOWFLAKE_ROLE')}}
+snowflake-warehouse: {{ env_var('SNOWFLAKE_WAREHOUSE')}}
+snowflake-database: {{ env_var('SNOWFLAKE_DATABASE')}}
+# tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment.
+change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY"
+create-change-history-table: true
+
+vars:
+  database_name: {{env_var('SNOWFLAKE_DATABASE')}}
+  schema_name: {{env_var('MY_TARGET_SCHEMA')}}
diff --git a/...mo_jinja/teardown_citibike_demo_jinja.sql → ...jinja/A__teardown_citibike_demo_jinja.sql b/...mo_jinja/teardown_citibike_demo_jinja.sql → ...jinja/A__teardown_citibike_demo_jinja.sql
diff --git a/demo/teardown/citibike_demo_jinja/schemachange-config.yml b/demo/teardown/citibike_demo_jinja/schemachange-config.yml
@@ -0,0 +1,16 @@
+config-version: 1
+
+root-folder: ./demo/teardown/{{ env_var('SCENARIO_NAME')}}
+
+snowflake-user: {{ env_var('SNOWFLAKE_USER')}}
+snowflake-account: {{ env_var('SNOWFLAKE_ACCOUNT')}}
+snowflake-role: {{ env_var('SNOWFLAKE_ROLE')}}
+snowflake-warehouse: {{ env_var('SNOWFLAKE_WAREHOUSE')}}
+snowflake-database: {{ env_var('SNOWFLAKE_DATABASE')}}
+# tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment.
+change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY"
+create-change-history-table: true
+
+vars:
+  database_name: {{env_var('SNOWFLAKE_DATABASE')}}
+  schema_name: {{env_var('MY_TARGET_SCHEMA')}}