diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 5dfb587c..c163d212 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -22,7 +22,7 @@ jobs: with: python-version: 3.7 - name: Cache pip - uses: actions/cache@v2 + uses: actions/cache@v3.0.2 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('requirements-linters.txt') }} @@ -51,7 +51,7 @@ jobs: with: python-version: 3.7 - name: Cache pip - uses: actions/cache@v2 + uses: actions/cache@v3.0.2 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('requirements-linters.txt') }} diff --git a/.github/workflows/unittest.yaml b/.github/workflows/unittest.yaml index e6317b71..17b647ff 100644 --- a/.github/workflows/unittest.yaml +++ b/.github/workflows/unittest.yaml @@ -20,7 +20,7 @@ jobs: with: python-version: 3.7 - name: Cache pip - uses: actions/cache@v2 + uses: actions/cache@v3.0.2 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('requirements-*.txt') }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 07d11c96..0e901e3a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,7 +39,7 @@ repos: hooks: - id: isort - repo: https://github.com/psf/black - rev: 22.1.0 + rev: 22.3.0 hooks: - id: black - repo: https://gitlab.com/pycqa/flake8 @@ -47,20 +47,20 @@ repos: hooks: - id: flake8 additional_dependencies: - - "flake8-bugbear==22.1.11" + - "flake8-bugbear==22.3.23" - "flake8-comprehensions==3.7.0" - "flake8-docstrings==1.6.0" - "flake8-fixme==1.1.1" - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.931 + rev: v0.942 hooks: - id: mypy additional_dependencies: - "mypy-boto3~=1.21" - - "types-PyYAML==6.0.4" - - "types-setuptools==57.4.9" - - "types-simplejson==3.17.3" - - "types-tabulate==0.8.5" + - "types-PyYAML==6.0.6" + - "types-setuptools==57.4.14" + - "types-simplejson==3.17.5" + - "types-tabulate==0.8.7" - "types-termcolor==1.1.3" - repo: https://github.com/shellcheck-py/shellcheck-py rev: v0.7.2.1 diff --git a/python/etl/config/default_settings.yaml b/python/etl/config/default_settings.yaml index 2e69448a..2e33fb90 100644 --- a/python/etl/config/default_settings.yaml +++ b/python/etl/config/default_settings.yaml @@ -16,7 +16,7 @@ "retriable_error_codes": "8001,15001,15005", "concurrent_load_idle_termination_seconds": 3600, "redshift": { - "relation_column_encoding": "ON" + "relation_column_encoding": "AUTO" } }, # Target (Redshift) cluster diff --git a/python/etl/data_warehouse.py b/python/etl/data_warehouse.py index 41ad7c33..652503b3 100755 --- a/python/etl/data_warehouse.py +++ b/python/etl/data_warehouse.py @@ -355,6 +355,7 @@ def initial_setup(with_user_creation=False, force=False, dry_run=False) -> None: ) etl.db.drop_and_create_database(conn, database_name, config.owner.name) + # TODO(tom): This fails if database is not actually created. with closing( etl.db.connection(config.dsn_admin_on_etl_db, autocommit=True, readonly=dry_run) ) as conn: diff --git a/python/etl/dialect/redshift.py b/python/etl/dialect/redshift.py index f4c34c65..1e536092 100644 --- a/python/etl/dialect/redshift.py +++ b/python/etl/dialect/redshift.py @@ -181,9 +181,7 @@ def add_auto_encoding(table_design: dict) -> None: def build_table_ddl(table_name: TableName, table_design: dict, is_temp=False) -> str: """Assemble the DDL of a table in a Redshift data warehouse.""" - if ( - etl.config.get_config_value("arthur_settings.redshift.relation_column_encoding") or "ON" - ) == "AUTO": + if etl.config.get_config_value("arthur_settings.redshift.relation_column_encoding", "ON") == "AUTO": add_auto_encoding(table_design) columns = build_columns(table_design["columns"], is_temp=is_temp) constraints = build_table_constraints(table_design) @@ -333,7 +331,7 @@ def copy_using_manifest( data_format, format_option, file_compression ) - compupdate = etl.config.get_config_value("arthur_settings.redshift.relation_column_encoding") or "ON" + compupdate = etl.config.get_config_value("arthur_settings.redshift.relation_column_encoding", "ON") if compupdate == "AUTO": compupdate = "OFF" diff --git a/requirements-linters.txt b/requirements-linters.txt index a0a3d7c2..9fc79ff2 100644 --- a/requirements-linters.txt +++ b/requirements-linters.txt @@ -1,18 +1,18 @@ # NOTE When changing versions here, be sure to keep .pre-commit-config.yaml consistent! -black==22.1.0 +black==22.3.0 flake8==4.0.1 -flake8-bugbear==22.1.11 +flake8-bugbear==22.3.23 flake8-comprehensions==3.7.0 flake8-docstrings==1.6.0 flake8-fixme==1.1.1 isort==5.10.1 -mypy==0.931 +mypy==0.942 mypy-boto3~=1.21 mypy-extensions==0.4.3 pre-commit==2.17.0 pyupgrade==2.31.0 -types-PyYAML==6.0.4 -types-setuptools==57.4.9 -types-simplejson==3.17.3 -types-tabulate==0.8.5 +types-PyYAML==6.0.6 +types-setuptools==57.4.14 +types-simplejson==3.17.5 +types-tabulate==0.8.7 types-termcolor==1.1.3 diff --git a/requirements.txt b/requirements.txt index bca398e1..42798c9e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ # This is the set of packages need to run the code. All versions are pinned. arrow==1.2.2 -boto3==1.21.13 +boto3==1.21.42 botocore~=1.24 funcy==1.17 jmespath==0.10.0 @@ -11,5 +11,5 @@ PyYAML==6.0 simplejson==3.17.6 tabulate==0.8.9 termcolor==1.1.0 -tqdm==4.63.0 +tqdm==4.64.0 watchtower==3.0.0 diff --git a/setup.py b/setup.py index 06bd82a2..12437743 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -ARTHUR_VERSION = "1.60.0" +ARTHUR_VERSION = "1.61.0" setup(