From 673f946579067609e34e2a65d872e66220884919 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Mon, 6 Nov 2023 15:32:46 -0500 Subject: [PATCH 1/3] Update issue templates (#206) * Update issue templates: Bug Report, Questions and Enhancement templates --- .github/ISSUE_TEMPLATE/bug_report.md | 30 +++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 20 +++++++++++++ .../question-or-clarification.md | 15 ++++++++++ 3 files changed, 65 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/ISSUE_TEMPLATE/question-or-clarification.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..da2d4323 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,30 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Schemachange (please complete the following information):** +- Version (e.g. 3.5.3): x.x.x + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..11fc491e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: enhancement +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/ISSUE_TEMPLATE/question-or-clarification.md b/.github/ISSUE_TEMPLATE/question-or-clarification.md new file mode 100644 index 00000000..278c64a6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question-or-clarification.md @@ -0,0 +1,15 @@ +--- +name: Question or Clarification +about: Need help with getting a functionality to work OR seeking workarounds for a + situation you are facing +title: How do I .... ? +labels: question, Workaround +assignees: '' + +--- + +** What are you trying to accomplish? ** +Describe what you are trying to accomplish with SchemaChange + +** What options have you tried so far ? ** +Share any steps, examples, solutions (include links, if possible) you have tried. From 41c2606b0be57270029d00cf4c2501745e8049d6 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Mon, 6 Nov 2023 17:44:47 -0500 Subject: [PATCH 2/3] Create CONTRIBUTING.md Initial workflow for contributing to schemachange repository --- .github/CONTRIBUTING.md | 70 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .github/CONTRIBUTING.md diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 00000000..e6a49e39 --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,70 @@ +# Contributing to schemachange + +## Reporting issues + +When reporting issues please include as much detail as possible about your +operating environment, schemachange version and python version. Whenever possible, please +also include a brief, self-contained code example that demonstrates the problem. + +We have included [issue templates for reporting bugs, requesting features and seeking clarifications.](https://github.com/Snowflake-Labs/schemachange/issues/new/choose) +Choose the appropriate issue template to contribute to the repository. + +## Contributing code + +Thanks for your interest in contributing code to schemachange! + ++ If this is your first time contributing to a project on GitHub, please read through our [guide to contributing to schemachange](guide-to-contributing-to-schemachange). ++ There are many online tutorials to help you [learn git](https://try.github.io/). For discussions of specific git workflows, see these discussions on [linux git workflow](https://www.mail-archive.com/dri-devel@lists.sourceforge.net/msg39091.html), and [ipython git workflow](https://mail.python.org/pipermail/ipython-dev/2010-October/005632.html). + +### Guide to contributing to schemachange + +1. If you are a first-time contributor + + Go to [Snowflake-Labs/Schemachange](https://github.com/Snowflake-Labs/schemachange) and click the "fork" button to create your own copy of the project. + + [Clone](https://github.com/git-guides/git-clone) the project to your local computer + + ```shell + git clone https://github.com/your-username/schemachange.git + + ``` + + + Change the directory + + ```shell + cd schemachange + + ``` + + Add upstream repository: + + ```shell + git remote add upstream https://github.com/Snowflake-Labs/schemachange + + ``` + + + Now, `git remote -v` will show two [remote](https://github.com/git-guides/git-remote) repositories named: + + `upstream`, which refers to the `schemachange` repository + + `origin`, which refers to your personal fork + + [Pull](https://github.com/git-guides/git-pull) the latest changes from upstream, including tags: + + ```shell + git checkout main + git pull upstream main --tags + + ``` + +3. Develop your contribution + + Create a branch for the features you want to work on. Since the branch name will appear in the merge message, use a sensible name such as 'update-build-library-dependencies': + + ```shell + git checkout -b update-build-library-dependencies + ``` + + + Commit locally as you progress ( [git add](https://github.com/git-guides/git-add) and [git commit](https://github.com/git-guides/git-commit) ). Use a properly formatted commit message. Be sure to document any changed behavior. +4. To submit your contribution + + [Push](https://github.com/git-guides/git-push) your changes back to your fork on GitHub + + ```shell + git push origin update-build-library-dependencies + + ``` + + + Go to GitHub. The new branch will show up with a green [Pull Request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests#initiating-the-pull-request) button. Make sure the title and message are clear, concise and self explanatory. Then click the button to submit it. From de2bca1d10e960bf6d9d6580b8ade664fda581aa Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Thu, 16 Nov 2023 15:49:49 -0500 Subject: [PATCH 3/3] Configure Snowflake Schema via Config (#210) (#211) * Update issue templates (#206) * Update issue templates: Bug Report, Questions and Enhancement templates * Create CONTRIBUTING.md Initial workflow for contributing to schemachange repository * Add support for specifying the schema via config * Documentation: Include details on configuring the default schema * Updated version and changelog --------- Co-authored-by: podung Co-authored-by: Tyler White <90005851+sfc-gh-twhite@users.noreply.github.com> --- CHANGELOG.md | 4 ++++ README.md | 12 ++++++++---- schemachange/cli.py | 20 ++++++++++++++------ setup.cfg | 2 +- tests/test_main.py | 3 +++ 5 files changed, 30 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a71a228..63e4b8fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. *The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).* +## [3.6.1] - 2023-11-15 +### Added +- Allow passing snowflake schema as config or CLI parameter + ## [3.6.0] - 2023-09-06 ### Changed - Fixed bug introduced in version 3.5.0 where the session state was not reset after a user script was run. This resulted in schemachange updates to the metadata table failing in some cases. schemachange will now reset the session back to the default settings after each user script is run diff --git a/README.md b/README.md index 7353dae0..2b5f0248 100644 --- a/README.md +++ b/README.md @@ -307,6 +307,9 @@ snowflake-warehouse: 'warehouse' # The name of the default database to use. Can be overridden in the change scripts. snowflake-database: null +# The name of the default schema to use. Can be overridden in the change scripts. +snowflake-schema: null + # Used to override the default name of the change history table (the default is METADATA.SCHEMACHANGE.CHANGE_HISTORY) change-history-table: null @@ -374,7 +377,7 @@ Schemachange supports a number of subcommands, it the subcommand is not provided #### deploy This is the main command that runs the deployment process. -`usage: schemachange deploy [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-m MODULES_FOLDER] [-a SNOWFLAKE_ACCOUNT] [-u SNOWFLAKE_USER] [-r SNOWFLAKE_ROLE] [-w SNOWFLAKE_WAREHOUSE] [-d SNOWFLAKE_DATABASE] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG]` +`usage: schemachange deploy [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-m MODULES_FOLDER] [-a SNOWFLAKE_ACCOUNT] [-u SNOWFLAKE_USER] [-r SNOWFLAKE_ROLE] [-w SNOWFLAKE_WAREHOUSE] [-d SNOWFLAKE_DATABASE] [-s SNOWFLAKE_SCHEMA] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG]` Parameter | Description --- | --- @@ -387,6 +390,7 @@ Parameter | Description -r SNOWFLAKE_ROLE, --snowflake-role SNOWFLAKE_ROLE | The name of the role to use -w SNOWFLAKE_WAREHOUSE, --snowflake-warehouse SNOWFLAKE_WAREHOUSE | The name of the default warehouse to use. Can be overridden in the change scripts. -d SNOWFLAKE_DATABASE, --snowflake-database SNOWFLAKE_DATABASE | The name of the default database to use. Can be overridden in the change scripts. +-s SNOWFLAKE_SCHEMA, --snowflake-schema SNOWFLAKE_SCHEMA | The name of the default schema to use. Can be overridden in the change scripts. -c CHANGE_HISTORY_TABLE, --change-history-table CHANGE_HISTORY_TABLE | Used to override the default name of the change history table (which is METADATA.SCHEMACHANGE.CHANGE_HISTORY) --vars VARS | Define values for the variables to replaced in change scripts, given in JSON format (e.g. '{"variable1": "value1", "variable2": "value2"}') --create-change-history-table | Create the change history table if it does not exist. The default is 'False'. @@ -429,13 +433,13 @@ In order to run schemachange you must have the following: schemachange is a single python script located at [schemachange/cli.py](schemachange/cli.py). It can be executed as follows: ``` -python schemachange/cli.py [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-a SNOWFLAKE_ACCOUNT] [-u SNOWFLAKE_USER] [-r SNOWFLAKE_ROLE] [-w SNOWFLAKE_WAREHOUSE] [-d SNOWFLAKE_DATABASE] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] [--oauth-config OUATH_CONFIG] +python schemachange/cli.py [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-a SNOWFLAKE_ACCOUNT] [-u SNOWFLAKE_USER] [-r SNOWFLAKE_ROLE] [-w SNOWFLAKE_WAREHOUSE] [-d SNOWFLAKE_DATABASE] [-s SNOWFLAKE_SCHEMA] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] [--oauth-config OUATH_CONFIG] ``` Or if installed via `pip`, it can be executed as follows: ``` -schemachange [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-a SNOWFLAKE_ACCOUNT] [-u SNOWFLAKE_USER] [-r SNOWFLAKE_ROLE] [-w SNOWFLAKE_WAREHOUSE] [-d SNOWFLAKE_DATABASE] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] [--oauth-config OUATH_CONFIG] +schemachange [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-a SNOWFLAKE_ACCOUNT] [-u SNOWFLAKE_USER] [-r SNOWFLAKE_ROLE] [-w SNOWFLAKE_WAREHOUSE] [-d SNOWFLAKE_DATABASE] [-s SNOWFLAKE_SCHEMA] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] [--oauth-config OUATH_CONFIG] ``` ## Getting Started with schemachange @@ -468,7 +472,7 @@ Here is a sample DevOps development lifecycle with schemachange: If your build agent has a recent version of python 3 installed, the script can be ran like so: ``` pip install schemachange --upgrade -schemachange [-h] [-f ROOT_FOLDER] -a SNOWFLAKE_ACCOUNT -u SNOWFLAKE_USER -r SNOWFLAKE_ROLE -w SNOWFLAKE_WAREHOUSE [-d SNOWFLAKE_DATABASE] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] [--oauth-config OUATH_CONFIG] +schemachange [-h] [-f ROOT_FOLDER] -a SNOWFLAKE_ACCOUNT -u SNOWFLAKE_USER -r SNOWFLAKE_ROLE -w SNOWFLAKE_WAREHOUSE [-d SNOWFLAKE_DATABASE] [-s SNOWFLAKE_SCHEMA] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] [--oauth-config OUATH_CONFIG] ``` Or if you prefer docker, set the environment variables and run like so: diff --git a/schemachange/cli.py b/schemachange/cli.py index 8ef83301..31745f43 100644 --- a/schemachange/cli.py +++ b/schemachange/cli.py @@ -21,7 +21,7 @@ #region Global Variables # metadata -_schemachange_version = '3.6.0' +_schemachange_version = '3.6.1' _config_file_name = 'schemachange-config.yml' _metadata_database_name = 'METADATA' _metadata_schema_name = 'SCHEMACHANGE' @@ -48,7 +48,8 @@ + "\nSNOWFLAKE_AUTHENTICATOR must be defined is using Oauth, OKTA or external Browser Authentication." _log_config_details = "Using Snowflake account {snowflake_account}\nUsing default role " \ + "{snowflake_role}\nUsing default warehouse {snowflake_warehouse}\nUsing default " \ - + "database {snowflake_database}" + + "database {snowflake_database}" \ + + "schema {snowflake_schema}" _log_ch_use = "Using change history table {database_name}.{schema_name}.{table_name} " \ + "(last altered {last_altered})" _log_ch_create = "Created change history table {database_name}.{schema_name}.{table_name}" @@ -211,6 +212,7 @@ class SnowflakeSchemachangeSession: + "'{status}','{user}',CURRENT_TIMESTAMP);" _q_set_sess_role = 'USE ROLE {role};' _q_set_sess_database = 'USE DATABASE {database};' + _q_set_sess_schema = 'USE SCHEMA {schema};' _q_set_sess_warehouse = 'USE WAREHOUSE {warehouse};' #endregion Query Templates @@ -225,7 +227,7 @@ def __init__(self, config): # Retreive Connection info from config dictionary self.conArgs = {"user": config['snowflake_user'],"account": config['snowflake_account'] \ ,"role": config['snowflake_role'],"warehouse": config['snowflake_warehouse'] \ - ,"database": config['snowflake_database'],"application": _snowflake_application_name \ + ,"database": config['snowflake_database'],"schema": config['snowflake_schema'], "application": _snowflake_application_name \ ,"session_parameters":session_parameters} self.oauth_config = config['oauth_config'] @@ -434,6 +436,8 @@ def reset_session(self): reset_query += self._q_set_sess_warehouse.format(**self.conArgs) + " " if self.conArgs['database']: reset_query += self._q_set_sess_database.format(**self.conArgs) + " " + if self.conArgs['schema']: + reset_query += self._q_set_sess_schema.format(**self.conArgs) + " " self.execute_snowflake_query(reset_query) @@ -629,7 +633,7 @@ def load_schemachange_config(config_file_path: str) -> Dict[str, Any]: return config def get_schemachange_config(config_file_path, root_folder, modules_folder, snowflake_account, \ - snowflake_user, snowflake_role, snowflake_warehouse, snowflake_database, \ + snowflake_user, snowflake_role, snowflake_warehouse, snowflake_database, snowflake_schema, \ change_history_table, vars, create_change_history_table, autocommit, verbose, \ dry_run, query_tag, oauth_config, **kwargs): @@ -640,6 +644,7 @@ def get_schemachange_config(config_file_path, root_folder, modules_folder, snowf "modules_folder":modules_folder, "snowflake_account":snowflake_account, \ "snowflake_user":snowflake_user, "snowflake_role":snowflake_role, \ "snowflake_warehouse":snowflake_warehouse, "snowflake_database":snowflake_database, \ + "snowflake_schema":snowflake_schema, \ "change_history_table":change_history_table, "vars":vars, \ "create_change_history_table":create_change_history_table, \ "autocommit":autocommit, "verbose":verbose, "dry_run":dry_run,\ @@ -654,7 +659,8 @@ def get_schemachange_config(config_file_path, root_folder, modules_folder, snowf # create Default values dictionary config_defaults = {"root_folder":os.path.abspath('.'), "modules_folder":None, \ "snowflake_account":None, "snowflake_user":None, "snowflake_role":None, \ - "snowflake_warehouse":None, "snowflake_database":None, "change_history_table":None, \ + "snowflake_warehouse":None, "snowflake_database":None, "snowflake_schema":None, \ + "change_history_table":None, \ "vars":{}, "create_change_history_table":False, "autocommit":False, "verbose":False, \ "dry_run":False , "query_tag":None , "oauth_config":None } #insert defualt values for items not populated @@ -816,6 +822,7 @@ def main(argv=sys.argv): parser_deploy.add_argument('-r', '--snowflake-role', type = str, help = 'The name of the default role to use', required = False) parser_deploy.add_argument('-w', '--snowflake-warehouse', type = str, help = 'The name of the default warehouse to use. Can be overridden in the change scripts.', required = False) parser_deploy.add_argument('-d', '--snowflake-database', type = str, help = 'The name of the default database to use. Can be overridden in the change scripts.', required = False) + parser_deploy.add_argument('-s', '--snowflake-schema', type = str, help = 'The name of the default schema to use. Can be overridden in the change scripts.', required = False) parser_deploy.add_argument('-c', '--change-history-table', type = str, help = 'Used to override the default name of the change history table (the default is METADATA.SCHEMACHANGE.CHANGE_HISTORY)', required = False) parser_deploy.add_argument('--vars', type = json.loads, help = 'Define values for the variables to replaced in change scripts, given in JSON format (e.g. {"variable1": "value1", "variable2": "value2"})', required = False) parser_deploy.add_argument('--create-change-history-table', action='store_true', help = 'Create the change history schema and table, if they do not exist (the default is False)', required = False) @@ -855,7 +862,8 @@ def main(argv=sys.argv): if args.subcommand == 'render': renderoveride = {"snowflake_account":None,"snowflake_user":None,"snowflake_role":None, \ "snowflake_warehouse":None,"snowflake_database":None,"change_history_table":None, \ - "create_change_history_table":None,"autocommit":None,"dry_run":None,"query_tag":None,"oauth_config":None } + "snowflake_schema":None,"create_change_history_table":None,"autocommit":None, \ + "dry_run":None,"query_tag":None,"oauth_config":None } schemachange_args.update(renderoveride) config = get_schemachange_config(**schemachange_args) diff --git a/setup.cfg b/setup.cfg index e168eb1c..b3f8ec01 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = schemachange -version = 3.6.0 +version = 3.6.1 description = A Database Change Management tool for Snowflake long_description = file: README.md long_description_content_type = text/markdown diff --git a/tests/test_main.py b/tests/test_main.py index 7865b166..39217fe3 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -15,6 +15,7 @@ 'snowflake_role':None, 'snowflake_warehouse': None, 'snowflake_database': None, + 'snowflake_schema': None, 'change_history_table': None, 'vars': {}, 'create_change_history_table': False, @@ -41,6 +42,8 @@ {**DEFAULT_CONFIG, 'snowflake_warehouse': 'warehouse'}), (["schemachange", "deploy", "--snowflake-database", "database"], {**DEFAULT_CONFIG, 'snowflake_database': 'database'}), + (["schemachange", "deploy", "--snowflake-schema", "schema"], + {**DEFAULT_CONFIG, "snowflake_schema": "schema"}), (["schemachange", "deploy", "--change-history-table", "db.schema.table"], {**DEFAULT_CONFIG, 'change_history_table': 'db.schema.table'}), (["schemachange", "deploy", "--vars", '{"var1": "val"}'],