diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml index 2e47ddf1..09325fb9 100644 --- a/.github/workflows/dev-pytest.yml +++ b/.github/workflows/dev-pytest.yml @@ -27,6 +27,8 @@ jobs: runs-on: ubuntu-latest strategy: fail-fast: false + matrix: + scenario-name: [ 'basics_demo', 'citibike_demo', 'citibike_demo_jinja'] steps: - uses: actions/checkout@v4 @@ -61,9 +63,9 @@ jobs: # echo "Tested SnowCLI connection successfully!" # echo "Set default connection" # snow --config-file config.toml connection set-default "myconnection" - - name: Test Basic Schemachange + - name: Test Schemachange - ${{ matrix.scenario-name }} env: - SCENARIO_NAME: basics_demo + SCENARIO_NAME: ${{ matrix.scenario-name }} run: | export MY_TARGET_SCHEMA="${SCENARIO_NAME}_${{ env.SCHEMACHANGE_BUILD_ID }}" echo "::group::Setting up ${MY_TARGET_SCHEMA}" diff --git a/demo/citibike/schemachange-config.yml b/demo/citibike/schemachange-config.yml deleted file mode 100644 index 6429bdf3..00000000 --- a/demo/citibike/schemachange-config.yml +++ /dev/null @@ -1,13 +0,0 @@ -config-version: 1 - -root-folder: ./demo/citibike - -snowflake-user: {{ env_var('SNOWFLAKE_USER')}} -snowflake-account: {{ env_var('SNOWFLAKE_ACCOUNT')}} -snowflake-role: SCHEMACHANGE_DEMO_DEPLOY -snowflake-warehouse: SCHEMACHANGE_DEMO_WH -snowflake-database: SCHEMACHANGE_DEMO -snowflake-schema: CITIBIKE_DEMO_{{ env_var('SCHEMACHANGE_BUILD_ID') }} -change-history-table: "SCHEMACHANGE_DEMO.SCHEMACHANGE.CITIBIKE_DEMO_{{ env_var('SCHEMACHANGE_BUILD_ID') }}_CHANGE_HISTORY" -create-change-history-table: true - diff --git a/demo/citibike/A__checks.sql b/demo/citibike_demo/A__checks.sql similarity index 100% rename from demo/citibike/A__checks.sql rename to demo/citibike_demo/A__checks.sql diff --git a/demo/citibike/R__checks.sql b/demo/citibike_demo/R__checks.sql similarity index 100% rename from demo/citibike/R__checks.sql rename to demo/citibike_demo/R__checks.sql diff --git a/demo/citibike/V1.1__initial_database_objects.sql b/demo/citibike_demo/V1.1__initial_database_objects.sql similarity index 91% rename from demo/citibike/V1.1__initial_database_objects.sql rename to demo/citibike_demo/V1.1__initial_database_objects.sql index 06db2155..d921dbcd 100644 --- a/demo/citibike/V1.1__initial_database_objects.sql +++ b/demo/citibike_demo/V1.1__initial_database_objects.sql @@ -1,7 +1,6 @@ -- Use Demo Database if it doesn't exist -USE DATABASE SCHEMACHANGE_DEMO; --- Set the database and schema context -USE SCHEMA SCHEMACHANGE_DEMO.CITIBIKE_DEMO; +use database {{ database_name }}; +use schema {{ schema_name }}; -- Create the file formats CREATE OR REPLACE FILE FORMAT CSV_NO_HEADER diff --git a/demo/citibike/V1.2__load_tables_from_s3.sql b/demo/citibike_demo/V1.2__load_tables_from_s3.sql similarity index 89% rename from demo/citibike/V1.2__load_tables_from_s3.sql rename to demo/citibike_demo/V1.2__load_tables_from_s3.sql index 0fbce4d0..8e272cac 100644 --- a/demo/citibike/V1.2__load_tables_from_s3.sql +++ b/demo/citibike_demo/V1.2__load_tables_from_s3.sql @@ -1,5 +1,6 @@ -- Set the database and schema context -USE SCHEMA SCHEMACHANGE_DEMO.CITIBIKE_DEMO; +use database {{ database_name }}; +use schema {{ schema_name }}; -- Load the trips data -- Trips data bucket content has been updated. diff --git a/demo/citibike_demo/schemachange-config.yml b/demo/citibike_demo/schemachange-config.yml new file mode 100644 index 00000000..603943c0 --- /dev/null +++ b/demo/citibike_demo/schemachange-config.yml @@ -0,0 +1,17 @@ +config-version: 1 + +root-folder: ./demo/{{ env_var('SCENARIO_NAME')}} + +snowflake-user: {{ env_var('SNOWFLAKE_USER')}} +snowflake-account: {{ env_var('SNOWFLAKE_ACCOUNT')}} +snowflake-role: {{ env_var('SNOWFLAKE_ROLE')}} +snowflake-warehouse: {{ env_var('SNOWFLAKE_WAREHOUSE')}} +snowflake-database: {{ env_var('SNOWFLAKE_DATABASE')}} +snowflake-schema: {{ env_var('MY_TARGET_SCHEMA')}} +change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.{{ env_var('MY_TARGET_SCHEMA')}}.CHANGE_HISTORY" +create-change-history-table: true + +vars: + database_name: {{env_var('SNOWFLAKE_DATABASE')}} + schema_name: {{env_var('MY_TARGET_SCHEMA')}} + diff --git a/demo/citibike_jinja/modules/create_stage.j2 b/demo/citibike_demo_jinja/modules/create_stage.j2 similarity index 100% rename from demo/citibike_jinja/modules/create_stage.j2 rename to demo/citibike_demo_jinja/modules/create_stage.j2 diff --git a/demo/citibike_demo_jinja/schemachange-config.yml b/demo/citibike_demo_jinja/schemachange-config.yml new file mode 100644 index 00000000..7ffa639a --- /dev/null +++ b/demo/citibike_demo_jinja/schemachange-config.yml @@ -0,0 +1,25 @@ +config-version: 1 + +root-folder: ./demo/{{ env_var('SCENARIO_NAME')}}/scripts +modules-folder: ./demo/{{ env_var('SCENARIO_NAME')}}/modules + +snowflake-user: {{ env_var('SNOWFLAKE_USER')}} +snowflake-account: {{ env_var('SNOWFLAKE_ACCOUNT')}} +snowflake-role: {{ env_var('SNOWFLAKE_ROLE')}} +snowflake-warehouse: {{ env_var('SNOWFLAKE_WAREHOUSE')}} +snowflake-database: {{ env_var('SNOWFLAKE_DATABASE')}} +snowflake-schema: {{ env_var('MY_TARGET_SCHEMA')}} +change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.{{ env_var('MY_TARGET_SCHEMA')}}.CHANGE_HISTORY" +create-change-history-table: true + +vars: + database_name: {{env_var('SNOWFLAKE_DATABASE')}} + schema_name: {{env_var('MY_TARGET_SCHEMA')}} + secrets: + # not a good example of secrets, just here to demo the secret filtering + trips_s3_bucket: s3://snowflake-workshop-lab/citibike-trips + weather_s3_bucket: s3://snowflake-workshop-lab/weather-nyc + + + + diff --git a/demo/citibike_jinja/scripts/V1.1__initial_database_objects.sql b/demo/citibike_demo_jinja/scripts/V1.1__initial_database_objects.sql similarity index 100% rename from demo/citibike_jinja/scripts/V1.1__initial_database_objects.sql rename to demo/citibike_demo_jinja/scripts/V1.1__initial_database_objects.sql diff --git a/demo/citibike_jinja/scripts/V1.2__load_tables_from_s3.sql b/demo/citibike_demo_jinja/scripts/V1.2__load_tables_from_s3.sql similarity index 100% rename from demo/citibike_jinja/scripts/V1.2__load_tables_from_s3.sql rename to demo/citibike_demo_jinja/scripts/V1.2__load_tables_from_s3.sql diff --git a/demo/citibike_jinja/schemachange-config.yml b/demo/citibike_jinja/schemachange-config.yml deleted file mode 100644 index 91fe076a..00000000 --- a/demo/citibike_jinja/schemachange-config.yml +++ /dev/null @@ -1,21 +0,0 @@ -config-version: 1 - -root-folder: ./demo/citibike_jinja/scripts -modules-folder: ./demo/citibike_jinja/modules - -snowflake-user: {{ env_var('SNOWFLAKE_USER')}} -snowflake-account: {{ env_var('SNOWFLAKE_ACCOUNT')}} -snowflake-role: SCHEMACHANGE_DEMO_DEPLOY -snowflake-warehouse: SCHEMACHANGE_DEMO_WH -snowflake-database: SCHEMACHANGE_DEMO -snowflake-schema: CITIBIKE_DEMO_JINJA_{{ env_var('SCHEMACHANGE_BUILD_ID') }} -change-history-table: "SCHEMACHANGE_DEMO.SCHEMACHANGE.CITIBIKE_DEMO_JNJA_{{ env_var('SCHEMACHANGE_BUILD_ID') }}_CHANGE_HISTORY" -create-change-history-table: true - -vars: - database_name: {{env_var('SNOWFLAKE_DATABASE', 'SCHEMACHANGE_DEMO')}} - schema_name: {{env_var('SNOWFLAKE_SCHEMA', 'CITIBIKE_DEMO_JINJA')}} - secrets: - # not a good example of secrets, just here to demo the secret filtering - trips_s3_bucket: s3://snowflake-workshop-lab/citibike-trips - weather_s3_bucket: s3://snowflake-workshop-lab/weather-nyc diff --git a/demo/setup/basics_demo/A__basics_demo.sql b/demo/setup/basics_demo/A__setup_basics_demo.sql similarity index 100% rename from demo/setup/basics_demo/A__basics_demo.sql rename to demo/setup/basics_demo/A__setup_basics_demo.sql diff --git a/demo/setup/citibike_demo/setup_citibike_demo.sql b/demo/setup/citibike_demo/A__setup_citibike_demo.sql similarity index 100% rename from demo/setup/citibike_demo/setup_citibike_demo.sql rename to demo/setup/citibike_demo/A__setup_citibike_demo.sql diff --git a/demo/setup/citibike_demo/schemachange-config.yml b/demo/setup/citibike_demo/schemachange-config.yml new file mode 100644 index 00000000..126b7ae7 --- /dev/null +++ b/demo/setup/citibike_demo/schemachange-config.yml @@ -0,0 +1,16 @@ +config-version: 1 + +root-folder: ./demo/setup/{{ env_var('SCENARIO_NAME')}} + +snowflake-user: {{ env_var('SNOWFLAKE_USER')}} +snowflake-account: {{ env_var('SNOWFLAKE_ACCOUNT')}} +snowflake-role: {{ env_var('SNOWFLAKE_ROLE')}} +snowflake-warehouse: {{ env_var('SNOWFLAKE_WAREHOUSE')}} +snowflake-database: {{ env_var('SNOWFLAKE_DATABASE')}} +# tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment. +change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY" +create-change-history-table: true + +vars: + database_name: {{env_var('SNOWFLAKE_DATABASE')}} + schema_name: {{env_var('MY_TARGET_SCHEMA')}} diff --git a/demo/setup/citibike_demo_jinja/setup_citibike_demo_jinja.sql b/demo/setup/citibike_demo_jinja/A__setup_citibike_demo_jinja.sql similarity index 100% rename from demo/setup/citibike_demo_jinja/setup_citibike_demo_jinja.sql rename to demo/setup/citibike_demo_jinja/A__setup_citibike_demo_jinja.sql diff --git a/demo/setup/citibike_demo_jinja/schemachange-config.yml b/demo/setup/citibike_demo_jinja/schemachange-config.yml new file mode 100644 index 00000000..126b7ae7 --- /dev/null +++ b/demo/setup/citibike_demo_jinja/schemachange-config.yml @@ -0,0 +1,16 @@ +config-version: 1 + +root-folder: ./demo/setup/{{ env_var('SCENARIO_NAME')}} + +snowflake-user: {{ env_var('SNOWFLAKE_USER')}} +snowflake-account: {{ env_var('SNOWFLAKE_ACCOUNT')}} +snowflake-role: {{ env_var('SNOWFLAKE_ROLE')}} +snowflake-warehouse: {{ env_var('SNOWFLAKE_WAREHOUSE')}} +snowflake-database: {{ env_var('SNOWFLAKE_DATABASE')}} +# tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment. +change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY" +create-change-history-table: true + +vars: + database_name: {{env_var('SNOWFLAKE_DATABASE')}} + schema_name: {{env_var('MY_TARGET_SCHEMA')}} diff --git a/demo/teardown/basics_demo/A__basics_demo.sql b/demo/teardown/basics_demo/A__teardown_basics_demo.sql similarity index 100% rename from demo/teardown/basics_demo/A__basics_demo.sql rename to demo/teardown/basics_demo/A__teardown_basics_demo.sql diff --git a/demo/teardown/citibike_demo/teardown_citibike_demo.sql b/demo/teardown/citibike_demo/A__teardown_citibike_demo.sql similarity index 100% rename from demo/teardown/citibike_demo/teardown_citibike_demo.sql rename to demo/teardown/citibike_demo/A__teardown_citibike_demo.sql diff --git a/demo/teardown/citibike_demo/schemachange-config.yml b/demo/teardown/citibike_demo/schemachange-config.yml new file mode 100644 index 00000000..74e4ffc5 --- /dev/null +++ b/demo/teardown/citibike_demo/schemachange-config.yml @@ -0,0 +1,16 @@ +config-version: 1 + +root-folder: ./demo/teardown/{{ env_var('SCENARIO_NAME')}} + +snowflake-user: {{ env_var('SNOWFLAKE_USER')}} +snowflake-account: {{ env_var('SNOWFLAKE_ACCOUNT')}} +snowflake-role: {{ env_var('SNOWFLAKE_ROLE')}} +snowflake-warehouse: {{ env_var('SNOWFLAKE_WAREHOUSE')}} +snowflake-database: {{ env_var('SNOWFLAKE_DATABASE')}} +# tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment. +change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY" +create-change-history-table: true + +vars: + database_name: {{env_var('SNOWFLAKE_DATABASE')}} + schema_name: {{env_var('MY_TARGET_SCHEMA')}} diff --git a/demo/teardown/citibike_demo_jinja/teardown_citibike_demo_jinja.sql b/demo/teardown/citibike_demo_jinja/A__teardown_citibike_demo_jinja.sql similarity index 100% rename from demo/teardown/citibike_demo_jinja/teardown_citibike_demo_jinja.sql rename to demo/teardown/citibike_demo_jinja/A__teardown_citibike_demo_jinja.sql diff --git a/demo/teardown/citibike_demo_jinja/schemachange-config.yml b/demo/teardown/citibike_demo_jinja/schemachange-config.yml new file mode 100644 index 00000000..74e4ffc5 --- /dev/null +++ b/demo/teardown/citibike_demo_jinja/schemachange-config.yml @@ -0,0 +1,16 @@ +config-version: 1 + +root-folder: ./demo/teardown/{{ env_var('SCENARIO_NAME')}} + +snowflake-user: {{ env_var('SNOWFLAKE_USER')}} +snowflake-account: {{ env_var('SNOWFLAKE_ACCOUNT')}} +snowflake-role: {{ env_var('SNOWFLAKE_ROLE')}} +snowflake-warehouse: {{ env_var('SNOWFLAKE_WAREHOUSE')}} +snowflake-database: {{ env_var('SNOWFLAKE_DATABASE')}} +# tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment. +change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY" +create-change-history-table: true + +vars: + database_name: {{env_var('SNOWFLAKE_DATABASE')}} + schema_name: {{env_var('MY_TARGET_SCHEMA')}}