Move pipeline to cloud composer 2

.flake8

@@ -0,0 +1,5 @@
+[flake8]
+ignore = E203, E266, E501, W503, F403, F401
+max-line-length = 120
+max-complexity = 20
+select = B,C,E,F,W,T4,B9

.github/workflows/main.yml

@@ -0,0 +1,76 @@
+name: Python application
+
+on: [pull_request]
+
+jobs:
+  build:
+    name: tests-pass
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+#    - name: Install dependencies
+#      run: |
+#        python -m pip install --upgrade pip
+#        pip install -r requirements.txt
+#    - name: Test with pytest
+#      run: |
+#        coverage run -m pytest tests
+#        coverage xml -o coverage/python.xml
+#    - name: Report python coverage
+#      uses: orgoro/coverage@v3
+#      with:
+#        coverageFile: coverage/python.xml
+#        token: ${{ secrets.GITHUB_TOKEN }}
+    # The next few steps only apply if you have javascript files
+    # - name: Setup node
+    #   uses: actions/setup-node@v3
+    #   with:
+    #     node-version: '18'
+    # - name: Test with jest
+    #   shell: bash
+    #   run: |
+    #     npm install
+    #     npm test -- --coverage --coverageReporters="json-summary" --coverageReporters="text" | tee ./coverage.txt
+    #   shell: bash
+    # - name: Report javascript coverage
+    #   uses: MishaKav/jest-coverage-comment@v1.0.20
+    #   with:
+    #     title: "JavaScript Coverage"
+    #     summary-title: "Summary"
+    #     coverage-title: "Modified Files"
+    #     github-token: ${{ secrets.GITHUB_TOKEN }}
+    #     report-only-changed-files: true
+    #     coverage-path: ./JS-FOLDER-NAME/coverage.txt
+    #     coverage-summary-path: ./JS-FOLDER-NAME/coverage/coverage-summary.json
+    #     coverage-path-prefix: JS-FOLDER-NAME/src/
+    # - name: Build output files
+    #   run: |
+    #     npm run build
+    # - name: Check links in built files
+    #   id: link_check
+    #   run: |
+    #     find public -name "*.js" -exec grep -Eo "(http|https):\/\/[^]\{\}\"'\\\(\)\> ]+" {} \; | sort -u > linklist.txt
+    #     printf '%s\n%s\n%s\n' "# LinkChecker URL list" "# <meta charset=\"UTF-8\">" "$(cat linklist.txt)" > linklist.txt
+    #     linkchecker linklist.txt --check-extern --ignore-url="https://.*\.fastly\.net/.*" --ignore-url="https://.*\.mapbox\..*" --ignore-url=".*//a\W.*" --ignore-url="http://(a|x|тест)" -o failures > output.txt || true
+    #     cat output.txt
+    #     echo "num_links=$(wc -l < output.txt | sed 's/^ *//g')" >> $GITHUB_OUTPUT
+    #     echo "links<<EOFdelimiter" >> $GITHUB_OUTPUT
+    #     echo "$(cat output.txt)" >> $GITHUB_OUTPUT
+    #     echo "EOFdelimiter" >> $GITHUB_OUTPUT
+    # - name: Edit PR comment about link checking
+    #   if: steps.link_check.outputs.num_links > 0
+    #   uses: thollander/actions-comment-pull-request@v2
+    #   with:
+    #     message: |
+    #       There are ${{ steps.link_check.outputs.num_links }} broken links. Check the code for these links:
+    #       ${{ steps.link_check.outputs.links }}
+    #     comment_tag: link_check_msg
+    - name: Run linting
+      run: |
+        pip install pre-commit
+        pre-commit run --all-files

.github/workflows/rebase-reminder.yml

@@ -0,0 +1,42 @@
+name: Rebase reminder
+on: [pull_request, pull_request_review]
+
+jobs:
+  build:
+    name: rebuild-reminder
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+    - name: Find behind count
+      id: behind_count
+      run: |
+        echo "behind_count=$(git rev-list --count ${{ github.event.pull_request.head.sha }}..${{ github.event.pull_request.base.sha }})" >> $GITHUB_OUTPUT
+    - name: Find ahead count
+      id: ahead_count
+      run: |
+        echo "ahead_count=$(git rev-list --count ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }})" >> $GITHUB_OUTPUT
+    - name: Find combined count
+      id: combined_count
+      run: |
+        echo "combined_count=$(expr ${{steps.behind_count.outputs.behind_count}} + ${{steps.ahead_count.outputs.ahead_count}})" >> $GITHUB_OUTPUT
+    - name: Edit PR comment - rebasing
+      if: steps.behind_count.outputs.behind_count > 0 && steps.combined_count.outputs.combined_count > 3
+      uses: thollander/actions-comment-pull-request@v1
+      with:
+        message: |
+          Needs rebasing :bangbang:
+          behind_count is ${{ steps.behind_count.outputs.behind_count }}
+          ahead_count is ${{ steps.ahead_count.outputs.ahead_count }}
+        comment_includes: 'rebasing'
+    - name: Edit PR comment - no rebasing
+      if: steps.behind_count.outputs.behind_count == 0 || steps.combined_count.outputs.combined_count <= 3
+      uses: thollander/actions-comment-pull-request@v1
+      with:
+        message: |
+          No need for rebasing :+1:
+          behind_count is ${{ steps.behind_count.outputs.behind_count }}
+          ahead_count is ${{ steps.ahead_count.outputs.ahead_count }}
+        comment_includes: 'rebasing'

.pre-commit-config.yaml

@@ -0,0 +1,37 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.1.0
+    hooks:
+    -   id: trailing-whitespace
+        exclude: "__snapshots__"
+    -   id: end-of-file-fixer
+    -   id: check-yaml
+    -   id: check-added-large-files
+    -   id: check-json
+# The next step only applies if you have javascript files.
+# There should be a package.json that installs eslint
+# (or eslint-config-react-app if you are using gatsby).
+# -   repo: https://github.com/pre-commit/mirrors-eslint
+#     rev: v8.24.0
+#     hooks:
+#     -   id: eslint
+-   repo: https://github.com/PyCQA/isort
+    rev: 5.11.5
+    hooks:
+    -   id: isort
+-   repo: https://github.com/ambv/black
+    rev: 22.3.0
+    hooks:
+    -   id: black
+        language_version: python3
+-   repo: https://github.com/PyCQA/flake8
+    rev: 4.0.1
+    hooks:
+    -   id: flake8
+#-   repo: https://github.com/sqlfluff/sqlfluff
+#    rev: 0.10.1
+#    hooks:
+#    -   id: sqlfluff-lint
+#    -   id: sqlfluff-fix

push_to_airflow.sh

@@ -1,4 +1,4 @@
-gsutil cp ror_dag.py gs://us-east1-production2023-cc1-01d75926-bucket/dags/
-gsutil -m cp -r ror_scripts gs://us-east1-production2023-cc1-01d75926-bucket/dags/
-gsutil -m cp -r schemas/* gs://us-east1-production2023-cc1-01d75926-bucket/dags/schemas/ror/
+gsutil cp ror_dag.py gs://us-east1-production-cc2-202-b42a7a54-bucket/dags/
+gsutil -m cp -r ror_scripts/* gs://airflow-data-exchange/ror/scripts/
+gsutil -m cp -r schemas/* gs://us-east1-production-cc2-202-b42a7a54-bucket/dags/schemas/ror/
 gsutil -m cp -r schemas gs://airflow-data-exchange/ror/

pyproject.toml

@@ -0,0 +1,14 @@
+[tool.black]
+py36 = true
+include = '\.pyi?$'
+exclude = '''
+/(
+\.git
+| \.venv
+| build
+| dist
+)/
+'''
+
+[tool.isort]
+profile = "black"