diff --git a/CHANGELOG.md b/CHANGELOG.md index c293235c..08109cf8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,11 @@ All notable changes to this project will be documented in this file. *The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).* +## [3.5.2] - 2023-02-12 +### Added +- Added filtering white list logic for V and R scripts during deploy. +- Added optional `--change-file-list` that accepts a list of files for the filtering logic. + ## [3.5.1] - 2023-02-11 ### Changed - Fixed a bug when handling default values from the command line with arguments defined as `action='store_true'` (create-change-history-table, auto-commit, verbose, and dry-run). diff --git a/README.md b/README.md index d4a71b20..662479ab 100644 --- a/README.md +++ b/README.md @@ -364,7 +364,7 @@ Schemachange supports a number of subcommands, it the subcommand is not provided #### deploy This is the main command that runs the deployment process. -`usage: schemachange deploy [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-m MODULES_FOLDER] [-a SNOWFLAKE_ACCOUNT] [-u SNOWFLAKE_USER] [-r SNOWFLAKE_ROLE] [-w SNOWFLAKE_WAREHOUSE] [-d SNOWFLAKE_DATABASE] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG]` +`usage: schemachange deploy [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-m MODULES_FOLDER] [-a SNOWFLAKE_ACCOUNT] [-u SNOWFLAKE_USER] [-r SNOWFLAKE_ROLE] [-w SNOWFLAKE_WAREHOUSE] [-d SNOWFLAKE_DATABASE] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] [-cfl CHANGE_FILE_LIST]` Parameter | Description --- | --- @@ -385,6 +385,7 @@ Parameter | Description --dry-run | Run schemachange in dry run mode. The default is 'False'. --query-tag | A string to include in the QUERY_TAG that is attached to every SQL statement executed. --oauth-config | Define values for the variables to Make Oauth Token requests (e.g. {"token-provider-url": "https//...", "token-request-payload": {"client_id": "GUID_xyz",...},... })' +-cfl --change-file-list | Comma delimited white list of files, full path. V and R scripts will only be deployed if found in the list. #### render This subcommand is used to render a single script to the console. It is intended to support the development and troubleshooting of script that use features from the jinja template engine. @@ -419,13 +420,13 @@ In order to run schemachange you must have the following: schemachange is a single python script located at [schemachange/cli.py](schemachange/cli.py). It can be executed as follows: ``` -python schemachange/cli.py [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-a SNOWFLAKE_ACCOUNT] [-u SNOWFLAKE_USER] [-r SNOWFLAKE_ROLE] [-w SNOWFLAKE_WAREHOUSE] [-d SNOWFLAKE_DATABASE] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] [--oauth-config OUATH_CONFIG] +python schemachange/cli.py [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-a SNOWFLAKE_ACCOUNT] [-u SNOWFLAKE_USER] [-r SNOWFLAKE_ROLE] [-w SNOWFLAKE_WAREHOUSE] [-d SNOWFLAKE_DATABASE] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] [--oauth-config OUATH_CONFIG] [-cfl CHANGE_FILE_LIST] ``` Or if installed via `pip`, it can be executed as follows: ``` -schemachange [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-a SNOWFLAKE_ACCOUNT] [-u SNOWFLAKE_USER] [-r SNOWFLAKE_ROLE] [-w SNOWFLAKE_WAREHOUSE] [-d SNOWFLAKE_DATABASE] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] [--oauth-config OUATH_CONFIG] +schemachange [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-a SNOWFLAKE_ACCOUNT] [-u SNOWFLAKE_USER] [-r SNOWFLAKE_ROLE] [-w SNOWFLAKE_WAREHOUSE] [-d SNOWFLAKE_DATABASE] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] [--oauth-config OUATH_CONFIG] [-cfl CHANGE_FILE_LIST] ``` ## Getting Started with schemachange diff --git a/schemachange/cli.py b/schemachange/cli.py index a8876da3..74f45f5e 100644 --- a/schemachange/cli.py +++ b/schemachange/cli.py @@ -21,7 +21,7 @@ #region Global Variables # metadata -_schemachange_version = '3.5.1' +_schemachange_version = '3.5.2' _config_file_name = 'schemachange-config.yml' _metadata_database_name = 'METADATA' _metadata_schema_name = 'SCHEMACHANGE' @@ -504,7 +504,7 @@ def deploy_command(config): print(_log_ch_max_version.format(max_published_version_display=max_published_version_display)) # Find all scripts in the root folder (recursively) and sort them correctly - all_scripts = get_all_scripts_recursively(config['root_folder'], config['verbose']) + all_scripts = get_all_scripts_recursively(config['root_folder'], config['verbose'], config['change_file_list']) all_script_names = list(all_scripts.keys()) # Sort scripts such that versioned scripts get applied first and then the repeatable ones. all_script_names_sorted = sorted_alphanumeric([script for script in all_script_names if script[0] == 'V']) \ @@ -541,7 +541,7 @@ def deploy_command(config): # check if there is a change of the checksum in the script if checksum_current == checksum_last: if config['verbose']: - print(_log_skip_r.format(script=script)) + print(_log_skip_r.format(**script)) scripts_skipped += 1 continue @@ -608,7 +608,7 @@ def load_schemachange_config(config_file_path: str) -> Dict[str, Any]: def get_schemachange_config(config_file_path, root_folder, modules_folder, snowflake_account, \ snowflake_user, snowflake_role, snowflake_warehouse, snowflake_database, \ change_history_table, vars, create_change_history_table, autocommit, verbose, \ - dry_run, query_tag, oauth_config, **kwargs): + dry_run, query_tag, oauth_config, change_file_list, **kwargs): # create cli override dictionary # Could refactor to just pass Args as a dictionary? @@ -620,7 +620,8 @@ def get_schemachange_config(config_file_path, root_folder, modules_folder, snowf "change_history_table":change_history_table, "vars":vars, \ "create_change_history_table":create_change_history_table, \ "autocommit":autocommit, "verbose":verbose, "dry_run":dry_run,\ - "query_tag":query_tag, "oauth_config":oauth_config} + "query_tag":query_tag, "oauth_config":oauth_config, \ + "change_file_list": change_file_list.split(',') if change_file_list else None} cli_inputs = {k:v for (k,v) in cli_inputs.items() if v} # load YAML inputs and convert kebabs to snakes @@ -658,10 +659,10 @@ def get_schemachange_config(config_file_path, root_folder, modules_folder, snowf return config -def get_all_scripts_recursively(root_directory, verbose): +def get_all_scripts_recursively(root_directory, verbose, change_file_list=None): all_files = dict() all_versions = list() - # Walk the entire directory structure recursively + # walk directory where file not found in change_file_list for (directory_path, directory_names, file_names) in os.walk(root_directory): for file_name in file_names: @@ -691,6 +692,10 @@ def get_all_scripts_recursively(root_directory, verbose): print("Ignoring non-change file " + file_full_path) continue + # check if file is in change_file_list, optional. + if change_file_list is not None and script_type != 'A' and file_full_path not in change_file_list: + continue + # script name is the filename without any jinja extension (file_part, extension_part) = os.path.splitext(file_name) if extension_part.upper() == ".JINJA": @@ -801,6 +806,7 @@ def main(argv=sys.argv): parser_deploy.add_argument('--dry-run', action='store_true', help = 'Run schemachange in dry run mode (the default is False)', required = False) parser_deploy.add_argument('--query-tag', type = str, help = 'The string to add to the Snowflake QUERY_TAG session value for each query executed', required = False) parser_deploy.add_argument('--oauth-config', type = json.loads, help = 'Define values for the variables to Make Oauth Token requests (e.g. {"token-provider-url": "https//...", "token-request-payload": {"client_id": "GUID_xyz",...},... })', required = False) + parser_deploy.add_argument('-cfl', '--change-file-list', type = str, help = 'Comma delimited list of files that should be included in the deploy check. A scripts will process as normal.', required = False) # TODO test CLI passing of args parser_render = subcommands.add_parser('render', description="Renders a script to the console, used to check and verify jinja output from scripts.") diff --git a/setup.cfg b/setup.cfg index f462b066..8fb69775 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = schemachange -version = 3.5.1 +version = 3.5.2 author = jamesweakley/jeremiahhansen description = A Database Change Management tool for Snowflake long_description = file: README.md