Merge remote-tracking branch 'origin/next'

.github/workflows/lint.yaml

@@ -22,7 +22,7 @@ jobs:
       with:
         python-version: 3.7
     - name: Cache pip
-      uses: actions/cache@v2
+      uses: actions/cache@v3.0.2
       with:
         path: ~/.cache/pip
         key: ${{ runner.os }}-pip-${{ hashFiles('requirements-linters.txt') }}
@@ -51,7 +51,7 @@ jobs:
       with:
         python-version: 3.7
     - name: Cache pip
-      uses: actions/cache@v2
+      uses: actions/cache@v3.0.2
       with:
         path: ~/.cache/pip
         key: ${{ runner.os }}-pip-${{ hashFiles('requirements-linters.txt') }}

.github/workflows/unittest.yaml

@@ -20,7 +20,7 @@ jobs:
       with:
         python-version: 3.7
     - name: Cache pip
-      uses: actions/cache@v2
+      uses: actions/cache@v3.0.2
       with:
         path: ~/.cache/pip
         key: ${{ runner.os }}-pip-${{ hashFiles('requirements-*.txt') }}

.pre-commit-config.yaml

@@ -39,28 +39,28 @@ repos:
     hooks:
       - id: isort
   - repo: https://github.com/psf/black
-    rev: 22.1.0
+    rev: 22.3.0
     hooks:
       - id: black
   - repo: https://gitlab.com/pycqa/flake8
     rev: 4.0.1
     hooks:
       - id: flake8
         additional_dependencies:
-          - "flake8-bugbear==22.1.11"
+          - "flake8-bugbear==22.3.23"
           - "flake8-comprehensions==3.7.0"
           - "flake8-docstrings==1.6.0"
           - "flake8-fixme==1.1.1"
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.931
+    rev: v0.942
     hooks:
       - id: mypy
         additional_dependencies:
           - "mypy-boto3~=1.21"
-          - "types-PyYAML==6.0.4"
-          - "types-setuptools==57.4.9"
-          - "types-simplejson==3.17.3"
-          - "types-tabulate==0.8.5"
+          - "types-PyYAML==6.0.6"
+          - "types-setuptools==57.4.14"
+          - "types-simplejson==3.17.5"
+          - "types-tabulate==0.8.7"
           - "types-termcolor==1.1.3"
   - repo: https://github.com/shellcheck-py/shellcheck-py
     rev: v0.7.2.1

python/etl/config/default_settings.yaml

@@ -16,7 +16,7 @@
     "retriable_error_codes": "8001,15001,15005",
     "concurrent_load_idle_termination_seconds": 3600,
     "redshift": {
-      "relation_column_encoding": "ON"
+      "relation_column_encoding": "AUTO"
     }
   },
   # Target (Redshift) cluster

python/etl/data_warehouse.py

@@ -355,6 +355,7 @@ def initial_setup(with_user_creation=False, force=False, dry_run=False) -> None:
             )
             etl.db.drop_and_create_database(conn, database_name, config.owner.name)
 
+    # TODO(tom): This fails if database is not actually created.
     with closing(
         etl.db.connection(config.dsn_admin_on_etl_db, autocommit=True, readonly=dry_run)
     ) as conn:

python/etl/dialect/redshift.py

@@ -181,9 +181,7 @@ def add_auto_encoding(table_design: dict) -> None:
 
 def build_table_ddl(table_name: TableName, table_design: dict, is_temp=False) -> str:
     """Assemble the DDL of a table in a Redshift data warehouse."""
-    if (
-        etl.config.get_config_value("arthur_settings.redshift.relation_column_encoding") or "ON"
-    ) == "AUTO":
+    if etl.config.get_config_value("arthur_settings.redshift.relation_column_encoding", "ON") == "AUTO":
         add_auto_encoding(table_design)
     columns = build_columns(table_design["columns"], is_temp=is_temp)
     constraints = build_table_constraints(table_design)
@@ -333,7 +331,7 @@ def copy_using_manifest(
         data_format, format_option, file_compression
     )
 
-    compupdate = etl.config.get_config_value("arthur_settings.redshift.relation_column_encoding") or "ON"
+    compupdate = etl.config.get_config_value("arthur_settings.redshift.relation_column_encoding", "ON")
     if compupdate == "AUTO":
         compupdate = "OFF"
 

requirements-linters.txt

@@ -1,18 +1,18 @@
 # NOTE When changing versions here, be sure to keep .pre-commit-config.yaml consistent!
-black==22.1.0
+black==22.3.0
 flake8==4.0.1
-flake8-bugbear==22.1.11
+flake8-bugbear==22.3.23
 flake8-comprehensions==3.7.0
 flake8-docstrings==1.6.0
 flake8-fixme==1.1.1
 isort==5.10.1
-mypy==0.931
+mypy==0.942
 mypy-boto3~=1.21
 mypy-extensions==0.4.3
 pre-commit==2.17.0
 pyupgrade==2.31.0
-types-PyYAML==6.0.4
-types-setuptools==57.4.9
-types-simplejson==3.17.3
-types-tabulate==0.8.5
+types-PyYAML==6.0.6
+types-setuptools==57.4.14
+types-simplejson==3.17.5
+types-tabulate==0.8.7
 types-termcolor==1.1.3

requirements.txt

@@ -1,6 +1,6 @@
 # This is the set of packages need to run the code. All versions are pinned.
 arrow==1.2.2
-boto3==1.21.13
+boto3==1.21.42
 botocore~=1.24
 funcy==1.17
 jmespath==0.10.0
@@ -11,5 +11,5 @@ PyYAML==6.0
 simplejson==3.17.6
 tabulate==0.8.9
 termcolor==1.1.0
-tqdm==4.63.0
+tqdm==4.64.0
 watchtower==3.0.0

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-ARTHUR_VERSION = "1.60.0"
+ARTHUR_VERSION = "1.61.0"
 
 
 setup(