R__ and V__ Script change list. Reopening after bugfix merge

CHANGELOG.md

@@ -3,6 +3,11 @@ All notable changes to this project will be documented in this file.
 
 *The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).*
 
+## [3.5.2] - 2023-02-12
+### Added
+- Added filtering white list logic for V and R scripts during deploy.
+- Added optional `--change-file-list` that accepts a list of files for the filtering logic.
+
 ## [3.5.1] - 2023-02-11
 ### Changed
 - Fixed a bug when handling default values from the command line with arguments defined as `action='store_true'` (create-change-history-table, auto-commit, verbose, and dry-run).

README.md

@@ -364,7 +364,7 @@ Schemachange supports a number of subcommands, it the subcommand is not provided
 #### deploy
 This is the main command that runs the deployment process.
 
-`usage: schemachange deploy [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-m MODULES_FOLDER] [-a SNOWFLAKE_ACCOUNT] [-u SNOWFLAKE_USER] [-r SNOWFLAKE_ROLE] [-w SNOWFLAKE_WAREHOUSE] [-d SNOWFLAKE_DATABASE] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG]`
+`usage: schemachange deploy [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-m MODULES_FOLDER] [-a SNOWFLAKE_ACCOUNT] [-u SNOWFLAKE_USER] [-r SNOWFLAKE_ROLE] [-w SNOWFLAKE_WAREHOUSE] [-d SNOWFLAKE_DATABASE] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] [-cfl CHANGE_FILE_LIST]`
 
 Parameter | Description
 --- | ---
@@ -385,6 +385,7 @@ Parameter | Description
 --dry-run | Run schemachange in dry run mode. The default is 'False'.
 --query-tag | A string to include in the QUERY_TAG that is attached to every SQL statement executed.
 --oauth-config | Define values for the variables to Make Oauth Token requests  (e.g. {"token-provider-url": "https//...", "token-request-payload": {"client_id": "GUID_xyz",...},... })'
+-cfl --change-file-list | Comma delimited white list of files, full path. V and R scripts will only be deployed if found in the list.
 
 #### render
 This subcommand is used to render a single script to the console. It is intended to support the development and troubleshooting of script that use features from the jinja template engine.
@@ -419,13 +420,13 @@ In order to run schemachange you must have the following:
 schemachange is a single python script located at [schemachange/cli.py](schemachange/cli.py). It can be executed as follows:
 
 ```
-python schemachange/cli.py [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-a SNOWFLAKE_ACCOUNT] [-u SNOWFLAKE_USER] [-r SNOWFLAKE_ROLE] [-w SNOWFLAKE_WAREHOUSE] [-d SNOWFLAKE_DATABASE] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] [--oauth-config OUATH_CONFIG]
+python schemachange/cli.py [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-a SNOWFLAKE_ACCOUNT] [-u SNOWFLAKE_USER] [-r SNOWFLAKE_ROLE] [-w SNOWFLAKE_WAREHOUSE] [-d SNOWFLAKE_DATABASE] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] [--oauth-config OUATH_CONFIG] [-cfl CHANGE_FILE_LIST]
 ```
 
 Or if installed via `pip`, it can be executed as follows:
 
 ```
-schemachange [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-a SNOWFLAKE_ACCOUNT] [-u SNOWFLAKE_USER] [-r SNOWFLAKE_ROLE] [-w SNOWFLAKE_WAREHOUSE] [-d SNOWFLAKE_DATABASE] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] [--oauth-config OUATH_CONFIG]
+schemachange [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-a SNOWFLAKE_ACCOUNT] [-u SNOWFLAKE_USER] [-r SNOWFLAKE_ROLE] [-w SNOWFLAKE_WAREHOUSE] [-d SNOWFLAKE_DATABASE] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] [--oauth-config OUATH_CONFIG] [-cfl CHANGE_FILE_LIST]
 ```
 
 ## Getting Started with schemachange

schemachange/cli.py

@@ -21,7 +21,7 @@
 
 #region Global Variables 
 # metadata
-_schemachange_version = '3.5.1'
+_schemachange_version = '3.5.2'
 _config_file_name = 'schemachange-config.yml'
 _metadata_database_name = 'METADATA'
 _metadata_schema_name = 'SCHEMACHANGE'
@@ -504,7 +504,7 @@ def deploy_command(config):
   print(_log_ch_max_version.format(max_published_version_display=max_published_version_display))
 
   # Find all scripts in the root folder (recursively) and sort them correctly
-  all_scripts = get_all_scripts_recursively(config['root_folder'], config['verbose'])
+  all_scripts = get_all_scripts_recursively(config['root_folder'], config['verbose'], config['change_file_list'])
   all_script_names = list(all_scripts.keys())
   # Sort scripts such that versioned scripts get applied first and then the repeatable ones.
   all_script_names_sorted =   sorted_alphanumeric([script for script in all_script_names if script[0] == 'V']) \
@@ -541,7 +541,7 @@ def deploy_command(config):
       # check if there is a change of the checksum in the script
       if checksum_current == checksum_last:
         if config['verbose']:
-          print(_log_skip_r.format(script=script))
+          print(_log_skip_r.format(**script))
         scripts_skipped += 1
         continue
 
@@ -608,7 +608,7 @@ def load_schemachange_config(config_file_path: str) -> Dict[str, Any]:
 def get_schemachange_config(config_file_path, root_folder, modules_folder, snowflake_account, \
   snowflake_user, snowflake_role, snowflake_warehouse, snowflake_database, \
   change_history_table, vars, create_change_history_table, autocommit, verbose, \
-  dry_run, query_tag, oauth_config, **kwargs):
+  dry_run, query_tag, oauth_config, change_file_list, **kwargs):
 
   # create cli override dictionary
   # Could refactor to just pass Args as a dictionary?
@@ -620,7 +620,8 @@ def get_schemachange_config(config_file_path, root_folder, modules_folder, snowf
     "change_history_table":change_history_table, "vars":vars, \
     "create_change_history_table":create_change_history_table, \
     "autocommit":autocommit, "verbose":verbose, "dry_run":dry_run,\
-    "query_tag":query_tag, "oauth_config":oauth_config}
+    "query_tag":query_tag, "oauth_config":oauth_config, \
+    "change_file_list": change_file_list.split(',') if change_file_list else None}
   cli_inputs = {k:v for (k,v) in cli_inputs.items() if v}
 
   # load YAML inputs and convert kebabs to snakes
@@ -658,10 +659,10 @@ def get_schemachange_config(config_file_path, root_folder, modules_folder, snowf
 
   return config
 
-def get_all_scripts_recursively(root_directory, verbose):
+def get_all_scripts_recursively(root_directory, verbose, change_file_list=None):
   all_files = dict()
   all_versions = list()
-  # Walk the entire directory structure recursively
+  # walk directory where file not found in change_file_list
   for (directory_path, directory_names, file_names) in os.walk(root_directory):
     for file_name in file_names:
 
@@ -691,6 +692,10 @@ def get_all_scripts_recursively(root_directory, verbose):
           print("Ignoring non-change file " + file_full_path)
         continue
 
+      # check if file is in change_file_list, optional.
+      if change_file_list is not None and script_type != 'A' and file_full_path not in change_file_list:
+        continue
+
       # script name is the filename without any jinja extension
       (file_part, extension_part) = os.path.splitext(file_name)
       if extension_part.upper() == ".JINJA":
@@ -801,6 +806,7 @@ def main(argv=sys.argv):
   parser_deploy.add_argument('--dry-run', action='store_true', help = 'Run schemachange in dry run mode (the default is False)', required = False)
   parser_deploy.add_argument('--query-tag', type = str, help = 'The string to add to the Snowflake QUERY_TAG session value for each query executed', required = False)
   parser_deploy.add_argument('--oauth-config', type = json.loads, help = 'Define values for the variables to Make Oauth Token requests  (e.g. {"token-provider-url": "https//...", "token-request-payload": {"client_id": "GUID_xyz",...},... })', required = False)
+  parser_deploy.add_argument('-cfl', '--change-file-list', type = str, help = 'Comma delimited list of files that should be included in the deploy check. A scripts will process as normal.', required = False)
    # TODO test CLI passing of args
 
   parser_render = subcommands.add_parser('render', description="Renders a script to the console, used to check and verify jinja output from scripts.")

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = schemachange
-version = 3.5.1
+version = 3.5.2
 author = jamesweakley/jeremiahhansen
 description = A Database Change Management tool for Snowflake
 long_description = file: README.md