From e740b6700205cd151ac828b8d2a98779723f0729 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Jan 2022 10:01:36 +0000 Subject: [PATCH 01/23] Bump psycopg2-binary from 2.9.2 to 2.9.3 Bumps [psycopg2-binary](https://github.com/psycopg/psycopg2) from 2.9.2 to 2.9.3. - [Release notes](https://github.com/psycopg/psycopg2/releases) - [Changelog](https://github.com/psycopg/psycopg2/blob/master/NEWS) - [Commits](https://github.com/psycopg/psycopg2/commits) --- updated-dependencies: - dependency-name: psycopg2-binary dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 07842058..e165f043 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ funcy==1.17 jmespath==0.10.0 jsonschema==3.2.0 pgpasslib==1.1.0 -psycopg2-binary==2.9.2 +psycopg2-binary==2.9.3 PyYAML==6.0 simplejson==3.17.6 tabulate==0.8.9 From 0e7f2b3d18086c6fe1b807608654ff0b7e9293f8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Jan 2022 10:01:48 +0000 Subject: [PATCH 02/23] Update sphinx requirement from ~=4.3 to ~=4.4 Updates the requirements on [sphinx](https://github.com/sphinx-doc/sphinx) to permit the latest version. - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/4.x/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v4.3.0...v4.4.0) --- updated-dependencies: - dependency-name: sphinx dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index b224d159..a34a973e 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -1,4 +1,4 @@ myst-parser~=0.16 -Sphinx~=4.3 +Sphinx~=4.4 sphinx-autodoc-typehints~=1.12 sphinx-book-theme~=0.1 From 605c558bb11a2454faefa834444a77043ca0818f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Jan 2022 10:02:17 +0000 Subject: [PATCH 03/23] Bump pre-commit from 2.16.0 to 2.17.0 Bumps [pre-commit](https://github.com/pre-commit/pre-commit) from 2.16.0 to 2.17.0. - [Release notes](https://github.com/pre-commit/pre-commit/releases) - [Changelog](https://github.com/pre-commit/pre-commit/blob/master/CHANGELOG.md) - [Commits](https://github.com/pre-commit/pre-commit/compare/v2.16.0...v2.17.0) --- updated-dependencies: - dependency-name: pre-commit dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-linters.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-linters.txt b/requirements-linters.txt index a8671cf0..daedcaf3 100644 --- a/requirements-linters.txt +++ b/requirements-linters.txt @@ -9,7 +9,7 @@ isort==5.10.1 mypy==0.931 mypy-boto3~=1.20 mypy-extensions==0.4.3 -pre-commit==2.16.0 +pre-commit==2.17.0 pyupgrade==2.31.0 types-PyYAML==6.0.3 types-setuptools==57.4.7 From 8bcd221e867329f3f691c3b55540387e355c5f86 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Jan 2022 10:02:10 +0000 Subject: [PATCH 04/23] Bump arrow from 1.2.1 to 1.2.2 Bumps [arrow](https://github.com/arrow-py/arrow) from 1.2.1 to 1.2.2. - [Release notes](https://github.com/arrow-py/arrow/releases) - [Changelog](https://github.com/arrow-py/arrow/blob/master/CHANGELOG.rst) - [Commits](https://github.com/arrow-py/arrow/compare/1.2.1...1.2.2) --- updated-dependencies: - dependency-name: arrow dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 894751b7..cbfb2ba7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # This is the set of packages need to run the code. All versions are pinned. -arrow==1.2.1 +arrow==1.2.2 boto3==1.20.33 botocore~=1.23 funcy==1.17 From 1c0f9b0c511c121400cbac15f6256d4ef1b2338b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Jan 2022 10:02:25 +0000 Subject: [PATCH 05/23] Bump black from 21.12b0 to 22.1.0 Bumps [black](https://github.com/psf/black) from 21.12b0 to 22.1.0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/commits/22.1.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements-linters.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-linters.txt b/requirements-linters.txt index a8671cf0..d3fb9864 100644 --- a/requirements-linters.txt +++ b/requirements-linters.txt @@ -1,5 +1,5 @@ # NOTE When changing versions here, be sure to keep .pre-commit-config.yaml consistent! -black==21.12b0 +black==22.1.0 flake8==4.0.1 flake8-bugbear==22.1.11 flake8-comprehensions==3.7.0 From 6f695a6a23cb47ef50ff24dcb86bf2ac06d184fc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Jan 2022 10:02:31 +0000 Subject: [PATCH 06/23] Bump watchtower from 2.1.1 to 3.0.0 Bumps [watchtower](https://github.com/kislyuk/watchtower) from 2.1.1 to 3.0.0. - [Release notes](https://github.com/kislyuk/watchtower/releases) - [Changelog](https://github.com/kislyuk/watchtower/blob/develop/Changes.rst) - [Commits](https://github.com/kislyuk/watchtower/compare/v2.1.1...v3.0.0) --- updated-dependencies: - dependency-name: watchtower dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 894751b7..ceefe03a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,4 +12,4 @@ simplejson==3.17.6 tabulate==0.8.9 termcolor==1.1.0 tqdm==4.62.3 -watchtower==2.1.1 +watchtower==3.0.0 From fb2e9f3014f4c16145fa6072057dd001113a146f Mon Sep 17 00:00:00 2001 From: ynaim94 Date: Wed, 16 Feb 2022 17:21:28 -0500 Subject: [PATCH 07/23] Add flags to on demand rebuild for load command --- .../text/ondemand_rebuild_pipeline.json | 12 +++++-- .../install_ondemand_rebuild_pipeline.sh | 33 ++++++++++++++++--- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/python/etl/templates/text/ondemand_rebuild_pipeline.json b/python/etl/templates/text/ondemand_rebuild_pipeline.json index 0db54c79..87dbc65a 100644 --- a/python/etl/templates/text/ondemand_rebuild_pipeline.json +++ b/python/etl/templates/text/ondemand_rebuild_pipeline.json @@ -133,7 +133,7 @@ "name": "Arthur Load (EC2)", "type": "ShellCommandActivity", "parent": { "ref": "ArthurCommandParent" }, - "command": "/tmp/redshift_etl/venv/bin/arthur.py --config /tmp/redshift_etl/config/ load --prolix --prefix ${object_store.s3.prefix} --concurrent-extract", + "command": "/tmp/redshift_etl/venv/bin/arthur.py --config /tmp/redshift_etl/config/ load --prolix --prefix ${object_store.s3.prefix} --concurrent-extract #{myExtraLoadFlags}", "dependsOn": { "ref": "ArthurTerminateSessions" } }, { @@ -177,9 +177,17 @@ "description": "How many hours to allow the pipeline to run before terminating it", "watermark": "6", "helpText": "How long can the pipeline run?" + }, + { + "id": "myExtraLoadFlags", + "type": "String", + "optional": "true", + "description": "Extra flags to add to the load command", + "helpText": "Extra flags to add to the load command other like skipping the use of staging " } ], "values": { - "myTimeout": "6" + "myTimeout": "6", + "myExtraLoadFlags": "" } } diff --git a/python/scripts/install_ondemand_rebuild_pipeline.sh b/python/scripts/install_ondemand_rebuild_pipeline.sh index a5098a91..a8ae97d0 100755 --- a/python/scripts/install_ondemand_rebuild_pipeline.sh +++ b/python/scripts/install_ondemand_rebuild_pipeline.sh @@ -3,17 +3,18 @@ CURRENT_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%S") DEFAULT_TIMEOUT=6 -if [[ $# -lt 1 || $# -gt 2 || "$1" = "-h" ]]; then +show_usage_and_exit() { cat < [timeout] +Usage: $(basename "$0") [timeout] [myExtraLoadFlags] Optional timeout should be the number of hours pipeline is allowed to run. Defaults to $DEFAULT_TIMEOUT. +Optional myExtraLoadFlags are passed to the rebuild load command USAGE exit 0 -fi +} set -o errexit -o nounset @@ -25,10 +26,33 @@ if [[ ! -d "$DEFAULT_CONFIG" ]]; then exit 1 fi +POSITIONAL_ARGS=() +EXTRA_LOAD_FLAGS=() + +while [[ $# -gt 0 ]]; do + case $1 in + -h) + show_usage_and_exit + ;; + -*) + EXTRA_LOAD_FLAGS+=("$1") # save flag + shift # past argument + ;; + *) + POSITIONAL_ARGS+=("$1") # save positional arg + shift # past argument + ;; + esac +done + +set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters + PROJ_BUCKET=$(arthur.py show_value object_store.s3.bucket_name) PROJ_ENVIRONMENT="$1" TIMEOUT="${2:-$DEFAULT_TIMEOUT}" +EXTRA_LOAD_FLAGS_STR="${EXTRA_LOAD_FLAGS[*]:-""}" + # Verify that this bucket/environment pair is set up on S3 BOOTSTRAP="s3://$PROJ_BUCKET/$PROJ_ENVIRONMENT/bin/bootstrap.sh" if ! aws s3 ls "$BOOTSTRAP" > /dev/null; then @@ -72,10 +96,11 @@ aws datapipeline put-pipeline-definition \ --pipeline-definition "file://$PIPELINE_DEFINITION_FILE" \ --parameter-values \ myTimeout="$TIMEOUT" \ + myExtraLoadFlags="${EXTRA_LOAD_FLAGS_STR}" \ --pipeline-id "$PIPELINE_ID" set +o xtrace echo echo "Your On-Demand Rebuild Pipeline ('$PIPELINE_ID') has been created!" echo "You can start the pipeline by running the following command:" -echo " aws datapipeline activate-pipeline --pipeline-id "$PIPELINE_ID"" +echo " aws datapipeline activate-pipeline --pipeline-id \"$PIPELINE_ID\"" From daa9c4280b1b932eea22be51cc75b09ec26e201b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Mar 2022 10:30:52 +0000 Subject: [PATCH 08/23] Bump actions/labeler from 3 to 4 Bumps [actions/labeler](https://github.com/actions/labeler) from 3 to 4. - [Release notes](https://github.com/actions/labeler/releases) - [Commits](https://github.com/actions/labeler/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/labeler dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/labeler.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index aaca80d5..873a35ea 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -7,6 +7,6 @@ jobs: apply-labels: runs-on: ubuntu-latest steps: - - uses: actions/labeler@v3 + - uses: actions/labeler@v4 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" From 333f90aedf422e02cb09fe1eb0cfef3c87f6f495 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Mar 2022 10:30:55 +0000 Subject: [PATCH 09/23] Bump actions/checkout from 2 to 3 Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 3. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yaml | 2 +- .github/workflows/lint.yaml | 4 ++-- .github/workflows/publish-latest-remote.yaml | 2 +- .github/workflows/publish-latest.yaml | 2 +- .github/workflows/unittest.yaml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/codeql-analysis.yaml b/.github/workflows/codeql-analysis.yaml index 41c44877..6d75daa3 100644 --- a/.github/workflows/codeql-analysis.yaml +++ b/.github/workflows/codeql-analysis.yaml @@ -29,7 +29,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Initialize CodeQL uses: github/codeql-action/init@v1 diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index bd986a83..25e319cc 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -16,7 +16,7 @@ jobs: steps: - name: Checkout out code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Python 3.7 uses: actions/setup-python@v2 with: @@ -45,7 +45,7 @@ jobs: steps: - name: Checkout out code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Python 3.7 uses: actions/setup-python@v2 with: diff --git a/.github/workflows/publish-latest-remote.yaml b/.github/workflows/publish-latest-remote.yaml index eed8d6ed..f2000650 100644 --- a/.github/workflows/publish-latest-remote.yaml +++ b/.github/workflows/publish-latest-remote.yaml @@ -19,7 +19,7 @@ jobs: steps: - name: Check out the repo - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Log in to the Container registry uses: docker/login-action@v1 diff --git a/.github/workflows/publish-latest.yaml b/.github/workflows/publish-latest.yaml index 27fe6080..3071175e 100644 --- a/.github/workflows/publish-latest.yaml +++ b/.github/workflows/publish-latest.yaml @@ -19,7 +19,7 @@ jobs: steps: - name: Check out the repo - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Log in to the Container registry uses: docker/login-action@v1 diff --git a/.github/workflows/unittest.yaml b/.github/workflows/unittest.yaml index 3e440666..0172d84a 100644 --- a/.github/workflows/unittest.yaml +++ b/.github/workflows/unittest.yaml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout out code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Python 3.7 uses: actions/setup-python@v2 with: From e4a3536b417b37600d440c638fcc0b6a92bd222c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Mar 2022 15:27:03 +0000 Subject: [PATCH 10/23] Update sphinx-book-theme requirement from ~=0.1 to ~=0.2 Updates the requirements on [sphinx-book-theme](https://github.com/executablebooks/sphinx-book-theme) to permit the latest version. - [Release notes](https://github.com/executablebooks/sphinx-book-theme/releases) - [Changelog](https://github.com/executablebooks/sphinx-book-theme/blob/master/CHANGELOG.md) - [Commits](https://github.com/executablebooks/sphinx-book-theme/compare/v0.1.0...v0.2.0) --- updated-dependencies: - dependency-name: sphinx-book-theme dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index a34a973e..3020fd89 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -1,4 +1,4 @@ myst-parser~=0.16 Sphinx~=4.4 sphinx-autodoc-typehints~=1.12 -sphinx-book-theme~=0.1 +sphinx-book-theme~=0.2 From f8bac908b323176b8c43aa1788de350e082fc3a5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Mar 2022 15:31:16 +0000 Subject: [PATCH 11/23] Bump boto3 from 1.20.33 to 1.21.13 Bumps [boto3](https://github.com/boto/boto3) from 1.20.33 to 1.21.13. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.20.33...1.21.13) --- updated-dependencies: - dependency-name: boto3 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cbfb2ba7..d5e27b5b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ # This is the set of packages need to run the code. All versions are pinned. arrow==1.2.2 -boto3==1.20.33 +boto3==1.21.13 botocore~=1.23 funcy==1.17 jmespath==0.10.0 From 0f459f1853e54340f8a625b1abc5faf0e914870d Mon Sep 17 00:00:00 2001 From: Tom Vogels Date: Mon, 7 Mar 2022 10:37:23 -0500 Subject: [PATCH 12/23] Update to black 22.1 --- .pre-commit-config.yaml | 2 +- python/etl/extract/database_extractor.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f408dd2a..e7bf6651 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,7 +39,7 @@ repos: hooks: - id: isort - repo: https://github.com/psf/black - rev: 21.12b0 + rev: 22.1.0 hooks: - id: black - repo: https://gitlab.com/pycqa/flake8 diff --git a/python/etl/extract/database_extractor.py b/python/etl/extract/database_extractor.py index 57b9e623..ba2b39de 100644 --- a/python/etl/extract/database_extractor.py +++ b/python/etl/extract/database_extractor.py @@ -37,7 +37,7 @@ def options_info(self) -> List[str]: def use_sampling_with_table(self, size: int) -> bool: """Return True iff option `--use-sampling` appeared and table is large enough (> 100MB).""" - return self.use_sampling and (size > 100 * 1024 ** 2) + return self.use_sampling and (size > 100 * 1024**2) def select_min_partition_size(self, size: int) -> int: """ @@ -49,8 +49,8 @@ def select_min_partition_size(self, size: int) -> int: * w/o sampling: 10MB """ if self.use_sampling_with_table(size): - return 100 * 1024 ** 2 - return 10 * 1024 ** 2 + return 100 * 1024**2 + return 10 * 1024**2 def maximize_partitions(self, table_size: int) -> int: """ From c01b8b83cc02da2090aa81e52395f18666b56b82 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Mar 2022 15:37:57 +0000 Subject: [PATCH 13/23] Bump tqdm from 4.62.3 to 4.63.0 Bumps [tqdm](https://github.com/tqdm/tqdm) from 4.62.3 to 4.63.0. - [Release notes](https://github.com/tqdm/tqdm/releases) - [Commits](https://github.com/tqdm/tqdm/compare/v4.62.3...v4.63.0) --- updated-dependencies: - dependency-name: tqdm dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 011ee241..3d9c9202 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,5 +11,5 @@ PyYAML==6.0 simplejson==3.17.6 tabulate==0.8.9 termcolor==1.1.0 -tqdm==4.62.3 +tqdm==4.63.0 watchtower==3.0.0 From c7000866361156d93fcfa285d9debec0853ae92e Mon Sep 17 00:00:00 2001 From: Tom Vogels Date: Mon, 7 Mar 2022 10:50:47 -0500 Subject: [PATCH 14/23] Update pre-commit --- .pre-commit-config.yaml | 2 +- requirements-linters.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f408dd2a..3ba3f610 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -56,7 +56,7 @@ repos: hooks: - id: mypy additional_dependencies: - - "mypy-boto3~=1.20" + - "mypy-boto3~=1.21" - "types-PyYAML==6.0.3" - "types-setuptools==57.4.7" - "types-simplejson==3.17.3" diff --git a/requirements-linters.txt b/requirements-linters.txt index daedcaf3..3354fdf1 100644 --- a/requirements-linters.txt +++ b/requirements-linters.txt @@ -7,7 +7,7 @@ flake8-docstrings==1.6.0 flake8-fixme==1.1.1 isort==5.10.1 mypy==0.931 -mypy-boto3~=1.20 +mypy-boto3~=1.21 mypy-extensions==0.4.3 pre-commit==2.17.0 pyupgrade==2.31.0 From f81a3362d75f48ee798fcfe02052d2661efc726f Mon Sep 17 00:00:00 2001 From: Tom Vogels Date: Mon, 7 Mar 2022 10:52:15 -0500 Subject: [PATCH 15/23] Update deps --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d5e27b5b..15c915a2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ # This is the set of packages need to run the code. All versions are pinned. arrow==1.2.2 boto3==1.21.13 -botocore~=1.23 +botocore~=1.24 funcy==1.17 jmespath==0.10.0 jsonschema==3.2.0 From d428161176df77fd9f97ae55c8d8d9f7290c0d5a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Mar 2022 15:53:57 +0000 Subject: [PATCH 16/23] Bump actions/setup-python from 2 to 3 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 2 to 3. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/lint.yaml | 4 ++-- .github/workflows/unittest.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 25e319cc..5dfb587c 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -18,7 +18,7 @@ jobs: - name: Checkout out code uses: actions/checkout@v3 - name: Set up Python 3.7 - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: 3.7 - name: Cache pip @@ -47,7 +47,7 @@ jobs: - name: Checkout out code uses: actions/checkout@v3 - name: Set up Python 3.7 - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: 3.7 - name: Cache pip diff --git a/.github/workflows/unittest.yaml b/.github/workflows/unittest.yaml index 0172d84a..e6317b71 100644 --- a/.github/workflows/unittest.yaml +++ b/.github/workflows/unittest.yaml @@ -16,7 +16,7 @@ jobs: - name: Checkout out code uses: actions/checkout@v3 - name: Set up Python 3.7 - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: 3.7 - name: Cache pip From 5e565420ba22d8762d74e01c1e0595370d5d6257 Mon Sep 17 00:00:00 2001 From: Tom Vogels Date: Mon, 7 Mar 2022 11:07:42 -0500 Subject: [PATCH 17/23] Revert "Update sphinx requirement from ~=4.3 to ~=4.4" This reverts commit 0e7f2b3d18086c6fe1b807608654ff0b7e9293f8. --- requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 3020fd89..61f10861 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -1,4 +1,4 @@ myst-parser~=0.16 -Sphinx~=4.4 +Sphinx~=4.3 sphinx-autodoc-typehints~=1.12 sphinx-book-theme~=0.2 From 9c6ffc55ca339e18dcc22866404f38581b498df7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Mar 2022 15:27:16 +0000 Subject: [PATCH 18/23] Bump types-pyyaml from 6.0.3 to 6.0.4 Bumps [types-pyyaml](https://github.com/python/typeshed) from 6.0.3 to 6.0.4. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-pyyaml dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-linters.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-linters.txt b/requirements-linters.txt index 75ceb33b..b2139180 100644 --- a/requirements-linters.txt +++ b/requirements-linters.txt @@ -11,7 +11,7 @@ mypy-boto3~=1.21 mypy-extensions==0.4.3 pre-commit==2.17.0 pyupgrade==2.31.0 -types-PyYAML==6.0.3 +types-PyYAML==6.0.4 types-setuptools==57.4.7 types-simplejson==3.17.3 types-tabulate==0.8.5 From 53a51efb6fc2fc866a02b6226ae0c8f70ebf4a10 Mon Sep 17 00:00:00 2001 From: Tom Vogels Date: Mon, 7 Mar 2022 11:07:12 -0500 Subject: [PATCH 19/23] Update pre-commit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 99e06227..188bc239 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -57,7 +57,7 @@ repos: - id: mypy additional_dependencies: - "mypy-boto3~=1.21" - - "types-PyYAML==6.0.3" + - "types-PyYAML==6.0.4" - "types-setuptools==57.4.7" - "types-simplejson==3.17.3" - "types-tabulate==0.8.5" From 021aadd29f9eb6da5706452c0c9fb80d1e1a735c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Mar 2022 16:39:30 +0000 Subject: [PATCH 20/23] Bump types-setuptools from 57.4.7 to 57.4.9 Bumps [types-setuptools](https://github.com/python/typeshed) from 57.4.7 to 57.4.9. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-setuptools dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-linters.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-linters.txt b/requirements-linters.txt index b2139180..a0a3d7c2 100644 --- a/requirements-linters.txt +++ b/requirements-linters.txt @@ -12,7 +12,7 @@ mypy-extensions==0.4.3 pre-commit==2.17.0 pyupgrade==2.31.0 types-PyYAML==6.0.4 -types-setuptools==57.4.7 +types-setuptools==57.4.9 types-simplejson==3.17.3 types-tabulate==0.8.5 types-termcolor==1.1.3 From 25d9db32a0e801d7460febd5ff1f333674be3f22 Mon Sep 17 00:00:00 2001 From: Tom Vogels Date: Mon, 7 Mar 2022 11:40:27 -0500 Subject: [PATCH 21/23] Update pre-commit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 188bc239..07d11c96 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -58,7 +58,7 @@ repos: additional_dependencies: - "mypy-boto3~=1.21" - "types-PyYAML==6.0.4" - - "types-setuptools==57.4.7" + - "types-setuptools==57.4.9" - "types-simplejson==3.17.3" - "types-tabulate==0.8.5" - "types-termcolor==1.1.3" From ea0804fe9a6a82ccc7ac3d2243d9783afb5c9d01 Mon Sep 17 00:00:00 2001 From: Tom Vogels Date: Mon, 7 Mar 2022 11:47:36 -0500 Subject: [PATCH 22/23] Bump version to 1.59 --- docs/source/conf.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 816c0679..46b1b246 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -23,7 +23,7 @@ author = "Data Engineering at Harry's" # TODO(tom): Extract from setup.py -version = "1.58" +version = "1.59" # -- General configuration --------------------------------------------------- diff --git a/setup.py b/setup.py index 8686e3bc..bcae69f1 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -ARTHUR_VERSION = "1.58.0" +ARTHUR_VERSION = "1.59.0" setup( From 090af77256536e2a18addb89718c664664b1dc34 Mon Sep 17 00:00:00 2001 From: ynaim94 Date: Wed, 9 Mar 2022 11:43:33 -0500 Subject: [PATCH 23/23] Remove unload from on-demand rebuild pipeline --- .../etl/templates/text/ondemand_rebuild_pipeline.json | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/python/etl/templates/text/ondemand_rebuild_pipeline.json b/python/etl/templates/text/ondemand_rebuild_pipeline.json index 87dbc65a..fe2cdd13 100644 --- a/python/etl/templates/text/ondemand_rebuild_pipeline.json +++ b/python/etl/templates/text/ondemand_rebuild_pipeline.json @@ -144,14 +144,6 @@ "command": "bash /tmp/redshift_etl/bin/sync_env.sh -y ${object_store.s3.bucket_name} ${object_store.s3.prefix} ${object_store.s3.prefix}/current", "dependsOn": { "ref": "ArthurLoad" } }, - { - "id": "ArthurUnload", - "name": "Arthur Unload (EC2)", - "type": "ShellCommandActivity", - "parent": {"ref": "ArthurCommandParent"}, - "command": "/tmp/redshift_etl/venv/bin/arthur.py --config /tmp/redshift_etl/config/ unload --keep-going --prolix --prefix ${object_store.s3.prefix}", - "dependsOn": {"ref": "ArthurLoad"} - }, { "id": "SendHealthCheckAfterEtl", "name": "Send Health Check After ETL (EC2)", @@ -159,8 +151,7 @@ "parent": { "ref": "ShellCommandParent" }, "command": "bash /tmp/redshift_etl/bin/send_health_check.sh", "dependsOn": [ - { "ref": "PublishAndBackup" }, - { "ref": "ArthurUnload" } + { "ref": "PublishAndBackup" } ], "onSuccess": { "ref": "SuccessNotification" }, "onFail": [