From d7f890610b9542049ab396464e928704731c1611 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Thu, 7 Dec 2023 11:26:46 -0500 Subject: [PATCH 01/15] Preparing for 3.6.2 Release --- CHANGELOG.md | 4 ++++ schemachange/cli.py | 8 ++++---- setup.cfg | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63e4b8fe..618cb8ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. *The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).* +## [3.6.2] - 2023-12-07 +### Changed +- Aligning with snowflake [identifier requirements](https://docs.snowflake.com/en/sql-reference/identifiers-syntax). + ## [3.6.1] - 2023-11-15 ### Added - Allow passing snowflake schema as config or CLI parameter diff --git a/schemachange/cli.py b/schemachange/cli.py index 31745f43..dc4ecbc5 100644 --- a/schemachange/cli.py +++ b/schemachange/cli.py @@ -21,7 +21,7 @@ #region Global Variables # metadata -_schemachange_version = '3.6.1' +_schemachange_version = '3.6.2' _config_file_name = 'schemachange-config.yml' _metadata_database_name = 'METADATA' _metadata_schema_name = 'SCHEMACHANGE' @@ -210,9 +210,9 @@ class SnowflakeSchemachangeSession: + "CHECKSUM, EXECUTION_TIME, STATUS, INSTALLED_BY, INSTALLED_ON) values ('{script_version}'," \ + "'{script_description}','{script_name}','{script_type}','{checksum}',{execution_time}," \ + "'{status}','{user}',CURRENT_TIMESTAMP);" - _q_set_sess_role = 'USE ROLE {role};' - _q_set_sess_database = 'USE DATABASE {database};' - _q_set_sess_schema = 'USE SCHEMA {schema};' + _q_set_sess_role = 'USE ROLE IDENTIFIER({role});' + _q_set_sess_database = 'USE DATABASE IDENTIFIER({database};' + _q_set_sess_schema = 'USE SCHEMA IDENTIFIER({schema};' _q_set_sess_warehouse = 'USE WAREHOUSE {warehouse};' #endregion Query Templates diff --git a/setup.cfg b/setup.cfg index b3f8ec01..582d3156 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = schemachange -version = 3.6.1 +version = 3.6.2 description = A Database Change Management tool for Snowflake long_description = file: README.md long_description_content_type = text/markdown From ff59dc654563c02942894fc7fb79842a24e7237c Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Wed, 3 Apr 2024 10:42:32 -0400 Subject: [PATCH 02/15] linting code using black formatter --- schemachange/cli.py | 4 ++-- tests/test_cli_misc.py | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/schemachange/cli.py b/schemachange/cli.py index ca36feaa..1a4563a9 100644 --- a/schemachange/cli.py +++ b/schemachange/cli.py @@ -886,7 +886,7 @@ def get_all_scripts_recursively(root_directory, verbose): all_files = dict() all_versions = list() # Walk the entire directory structure recursively - for (directory_path, directory_names, file_names) in os.walk(root_directory): + for directory_path, directory_names, file_names in os.walk(root_directory): for file_name in file_names: file_full_path = os.path.join(directory_path, file_name) @@ -1008,7 +1008,7 @@ def inner_extract_dictionary_secrets( extracted_secrets: Set[str] = set() if dictionary: - for (key, value) in dictionary.items(): + for key, value in dictionary.items(): if isinstance(value, dict): if key == "secrets": extracted_secrets = ( diff --git a/tests/test_cli_misc.py b/tests/test_cli_misc.py index b5dd2e75..6924e0b1 100644 --- a/tests/test_cli_misc.py +++ b/tests/test_cli_misc.py @@ -123,12 +123,11 @@ def test_get_change_history_table_details_given__acceptable_values_produces_full ): assert schemachange.cli.get_change_history_table_details(cht) == expected + @pytest.mark.parametrize( "cht", [("fifth.fourth.third.two.one"), ("fourth.third.two.one")] ) -def test_get_change_history_table_details_given__unacceptable_values_raises_error( - cht -): +def test_get_change_history_table_details_given__unacceptable_values_raises_error(cht): with pytest.raises(ValueError) as e: schemachange.cli.get_change_history_table_details(cht) From 7456bc3afabe1f5aab75edd232bacc734780f867 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Wed, 3 Apr 2024 11:00:13 -0400 Subject: [PATCH 03/15] Delegate validation to identifier function in snowflake --- schemachange/cli.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/schemachange/cli.py b/schemachange/cli.py index 1a4563a9..008165f7 100644 --- a/schemachange/cli.py +++ b/schemachange/cli.py @@ -267,10 +267,10 @@ class SnowflakeSchemachangeSession: + "'{script_description}','{script_name}','{script_type}','{checksum}',{execution_time}," + "'{status}','{user}',CURRENT_TIMESTAMP);" ) - _q_set_sess_role = "USE ROLE {role};" - _q_set_sess_database = "USE DATABASE {database};" - _q_set_sess_schema = "USE SCHEMA {schema};" - _q_set_sess_warehouse = "USE WAREHOUSE {warehouse};" + _q_set_sess_role = "USE ROLE IDENTIFIER({role});" + _q_set_sess_database = "USE DATABASE IDENTIFIER({database});" + _q_set_sess_schema = "USE SCHEMA IDENTIFIER({schema});" + _q_set_sess_warehouse = "USE WAREHOUSE IDENTIFIER({warehouse});" # endregion Query Templates def __init__(self, config): From 8a4942ef8a1c7a384cff157b1ce8d780197c29aa Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Wed, 3 Apr 2024 20:35:24 -0400 Subject: [PATCH 04/15] Leverage version number from setup.cfg --- schemachange/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/schemachange/cli.py b/schemachange/cli.py index 008165f7..4e29474f 100644 --- a/schemachange/cli.py +++ b/schemachange/cli.py @@ -14,6 +14,7 @@ import requests import snowflake.connector import yaml +import setuptools from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization from jinja2.loaders import BaseLoader @@ -21,7 +22,7 @@ # region Global Variables # metadata -_schemachange_version = "3.6.1" +_schemachange_version = setuptools.config.read_configuration('setup.cfg')['metadata']['version'] _config_file_name = "schemachange-config.yml" _metadata_database_name = "METADATA" _metadata_schema_name = "SCHEMACHANGE" From 411b61fdda9533d79097e737f5dec2b423167e72 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Wed, 3 Apr 2024 20:38:37 -0400 Subject: [PATCH 05/15] Updated test case for version check --- tests/test_cli_misc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_cli_misc.py b/tests/test_cli_misc.py index 6924e0b1..ff466850 100644 --- a/tests/test_cli_misc.py +++ b/tests/test_cli_misc.py @@ -2,8 +2,8 @@ import pytest -def test_cli_given__schemachange_version_change(): - assert schemachange.cli._schemachange_version == "3.6.1" +def test_cli_given__schemachange_version_change_updated_in_setup_config_file(): + assert schemachange.cli._schemachange_version == "3.6.2" def test_cli_given__constants_exist(): From 7e86d5527f016e1f29b0ae59bde5b79c3c0037e7 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Thu, 4 Apr 2024 08:19:30 -0400 Subject: [PATCH 06/15] Based on ruff and black compatibility Using only Ruff and skipping black to keep one formatter. --- .pre-commit-config.yaml | 12 +----------- schemachange/cli.py | 3 +-- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2ff9c8c4..443446e6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,14 +22,4 @@ repos: - id: ruff args: [ --fix ] # Run the formatter. - - id: ruff-format - # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.3.0 - hooks: - - id: black - # It is recommended to specify the latest version of Python - # supported by your project here, or alternatively use - # pre-commit's default_language_version, see - # https://pre-commit.com/#top_level-default_language_version - language_version: python3.11 + - id: ruff-format \ No newline at end of file diff --git a/schemachange/cli.py b/schemachange/cli.py index 4e29474f..201e7820 100644 --- a/schemachange/cli.py +++ b/schemachange/cli.py @@ -14,7 +14,6 @@ import requests import snowflake.connector import yaml -import setuptools from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization from jinja2.loaders import BaseLoader @@ -22,7 +21,7 @@ # region Global Variables # metadata -_schemachange_version = setuptools.config.read_configuration('setup.cfg')['metadata']['version'] +_schemachange_version = "3.6.2" _config_file_name = "schemachange-config.yml" _metadata_database_name = "METADATA" _metadata_schema_name = "SCHEMACHANGE" From 740356cadfe280f50404530cb2fb3bb92a21f4c9 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Thu, 4 Apr 2024 08:41:11 -0400 Subject: [PATCH 07/15] Updating github Actions to latest versions --- .github/workflows/dependency-review.yml | 2 +- .github/workflows/pytest.yml | 7 ++++--- .github/workflows/python-publish.yml | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index b9945082..b9d6d20f 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -11,4 +11,4 @@ jobs: - name: 'Checkout Repository' uses: actions/checkout@v4 - name: 'Dependency Review' - uses: actions/dependency-review-action@v3 + uses: actions/dependency-review-action@v4 diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 0ba747d4..58d3cddf 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -5,16 +5,17 @@ on: [push] jobs: build: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: + os: ["ubuntu-latest", "macos-latest", "windows-latest"] python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + - name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }} + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Update pip diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index bdaab28a..15e93dde 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -21,9 +21,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: '3.x' - name: Install dependencies From d12df867e382fc105460d53d9c4f99fb2705166a Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Thu, 4 Apr 2024 09:03:35 -0400 Subject: [PATCH 08/15] experimenting with precommit --- .pre-commit-config.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 443446e6..bf54e385 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,10 +2,17 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.2.0 + rev: v4.5.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer + - id: check-ast + - id: check-case-conflict + - id: check-docstring-first + - id: check-merge-conflict + - id: check-toml + - id: check-yaml + - id: name-tests-test - repo: https://github.com/asottile/setup-cfg-fmt rev: v2.4.0 hooks: From 95af260824a44871412462fe14f81b0966a62a2a Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Thu, 4 Apr 2024 09:27:32 -0400 Subject: [PATCH 09/15] Limiting testing on PRs to main and dev --- .github/workflows/pytest.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 58d3cddf..7afd88e1 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -1,6 +1,11 @@ name: pytest -on: [push] +on: + pull_request: + branches: + - master + - dev + jobs: build: From d9e4599643777211149d13bab59865c0584f0ed8 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Thu, 4 Apr 2024 09:31:25 -0400 Subject: [PATCH 10/15] trying dynamic versioning --- pyproject.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 9ac9b912..07871ec6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,3 +4,9 @@ requires = [ "wheel", ] build-backend = "setuptools.build_meta" + +[project] +dynamic = ["version"] + +[tool.setuptools.dynamic] +version = {attr = "schemachange.VERSION"} \ No newline at end of file From 7f9de9843a3af4f28db5cd5b4928ab991d8de3de Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Thu, 4 Apr 2024 10:43:16 -0400 Subject: [PATCH 11/15] getting latest actions versions --- .github/workflows/dev-pytest.yml | 32 ++++++++++++++++++++++++++++++++ .github/workflows/pytest.yml | 6 ++---- 2 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/dev-pytest.yml diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml new file mode 100644 index 00000000..fda654c9 --- /dev/null +++ b/.github/workflows/dev-pytest.yml @@ -0,0 +1,32 @@ +name: dev-pytest + +on: + push: + branches: + - dev + + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11"] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Update pip + run: | + python -m pip install --upgrade pip + - name: Install dependencies + run: | + python -m pip install -e .[dev] + - name: Test with pytest + run: | + pytest diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 7afd88e1..e52f5108 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -1,11 +1,9 @@ -name: pytest +name: master-pytest on: - pull_request: + push: branches: - master - - dev - jobs: build: From e51716d63769bc3f8641ba17f9aa7a3647c7f7e5 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Thu, 4 Apr 2024 13:08:13 -0400 Subject: [PATCH 12/15] limiting dev test to a single python version on actions. --- .github/workflows/dev-pytest.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml index fda654c9..177d2f6f 100644 --- a/.github/workflows/dev-pytest.yml +++ b/.github/workflows/dev-pytest.yml @@ -4,7 +4,9 @@ on: push: branches: - dev - + + workflow_dispatch: + jobs: build: @@ -12,15 +14,13 @@ jobs: runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python 3.11 uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: "3.11" - name: Update pip run: | python -m pip install --upgrade pip From 1c93a7e254ab244940f1cdb437aa491a232773f0 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Wed, 10 Apr 2024 23:57:09 -0400 Subject: [PATCH 13/15] Fixing pyproject.toml --- pyproject.toml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 07871ec6..4ba14420 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,10 +3,4 @@ requires = [ "setuptools >= 40.9.0", "wheel", ] -build-backend = "setuptools.build_meta" - -[project] -dynamic = ["version"] - -[tool.setuptools.dynamic] -version = {attr = "schemachange.VERSION"} \ No newline at end of file +build-backend = "setuptools.build_meta" \ No newline at end of file From cc8cc02e0b1d405a2adedcec1bae16857da09caa Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Wed, 10 Apr 2024 23:59:33 -0400 Subject: [PATCH 14/15] Adding github manual execution option in dev --- .github/workflows/dev-pytest.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml index 02ed99f6..3918a731 100644 --- a/.github/workflows/dev-pytest.yml +++ b/.github/workflows/dev-pytest.yml @@ -8,6 +8,8 @@ on: pull_request: types: [opened, reopened] + workflow_dispatch: + jobs: build: From 543faf9ed1612b1bf8e8a1b735a235e44a80e560 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Thu, 11 Apr 2024 00:01:14 -0400 Subject: [PATCH 15/15] Setting up github actions --- .github/workflows/dev-pytest.yml | 2 +- .github/workflows/{pytest.yml => master-pytest.yml} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{pytest.yml => master-pytest.yml} (100%) diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml index 3918a731..75f0ef15 100644 --- a/.github/workflows/dev-pytest.yml +++ b/.github/workflows/dev-pytest.yml @@ -1,4 +1,4 @@ -name: pytest +name: dev-pytest on: push: diff --git a/.github/workflows/pytest.yml b/.github/workflows/master-pytest.yml similarity index 100% rename from .github/workflows/pytest.yml rename to .github/workflows/master-pytest.yml