Merge pull request #1 from CIAT-DAPA/develop

Develop

.github/workflows/pipeline.yml

@@ -0,0 +1,116 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Devops FORAGE_ETL
+
+on:
+  push:
+    branches: ["stage"]
+    tags:
+      - "v*"
+
+permissions:
+  contents: read
+
+jobs:
+  # ------- START ORM PROCCESS -------- #
+
+  TestETL:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -el {0}
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Conda
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          activate-environment: Biomass_Pipeline
+          environment-file: ./src/environment.yml
+          python-version: 3.12
+          auto-activate-base: false
+
+      - name: Run Tests
+        run: |
+          python -m unittest discover -s ./src/tests/ -p 'test_*.py'
+  # ------- END ORM PROCCESS -------- #
+  # ------- START MERGE PROCCESS -------- #
+
+  MergeMainETL:
+    needs: TestETL
+    name: Merge Stage with Main
+    permissions: write-all
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+
+      - name: Merge stage -> main
+        uses: devmasx/merge-branch@master
+        with:
+          type: now
+          head_to_merge: ${{ github.ref }}
+          target_branch: main
+          github_token: ${{ github.token }}
+
+  # ------- END MERGE PROCCESS -------- #
+
+  # ------- START RELEASE PROCCESS -------- #
+
+  PostRelease:
+    needs: MergeMainETL
+    name: Create Release
+    runs-on: ubuntu-latest
+    permissions: write-all
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: "0"
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+      # API Zip
+      - name: Zip artifact for deployment
+        run: zip releaseForageEtl.zip ./src/* -r
+      # Upload Artifacts
+      - name: Upload Api artifact for deployment job
+        uses: actions/upload-artifact@v3
+        with:
+          name: ForageEtl
+          path: releaseForageEtl.zip
+      # Generate Tagname
+      - name: Generate Tagname for release
+        id: taggerDryRun
+        uses: anothrNick/github-tag-action@1.61.0
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          WITH_V: true
+          DRY_RUN: true
+          DEFAULT_BUMP: patch
+          RELEASE_BRANCHES: stage,main
+          BRANCH_HISTORY: last
+      # Create release
+      - name: Create Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        with:
+          tag_name: ${{ steps.taggerDryRun.outputs.new_tag }}
+          release_name: Release ${{ steps.taggerDryRun.outputs.new_tag }}
+          #body_path: ./body.md
+          body: ${{ github.event.head_commit.message }}
+          draft: false
+          prerelease: false
+      # Upload Assets to release
+      - name: Upload Release Asset API
+        id: upload-api-release-asset
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
+          asset_path: ./releaseForageEtl.zip
+          asset_name: releaseForageEtl.zip
+          asset_content_type: application/zip
+# ------- END RELEASE PROCCESS -------- #

README.md

@@ -49,5 +49,4 @@ You will need a private_key.json file associated with service_account placed in
 
 ```
 
-There is an sh file to run the scripts run_scripts.sh I recommend setting up a root cron job to run it.
-Before you run it with cron you will need to chmod to 755 so it can be executed properly.
+There is master_script.py to run all the other scripts.

src/codes/data_extraction.py

@@ -20,16 +20,18 @@
 
 
 # Constants
-filepath = '../data.json'
+project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+filepath = os.path.join(project_root, 'data.json')
+
 
 with open(filepath) as f:
     data = json.load(f)
 
 
-private_key =  '../private_key.json'
-outputs_path = '../outputs'
-inputs_path = '../inputs'
-data_path='../data'
+private_key = os.path.join(project_root, 'private_key.json')
+outputs_path = os.path.join(project_root, 'outputs')
+inputs_path = os.path.join(project_root, 'inputs')
+data_path = os.path.join(project_root, 'data')
 layers_path = os.path.join(data_path, "layers")
 biomass_path = os.path.join(layers_path, "biomass_et")
 grid_points=os.path.join(inputs_path,  "grid_points.xlsx")

src/codes/gwr_model.py

@@ -12,13 +12,9 @@
 from codes.funcs import *
 
 # Constants
-filepath = '../data.json'
+project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 
-with open(filepath) as f:
-    data = json.load(f)
-
-
-outputs_path = '../outputs'
+outputs_path = os.path.join(project_root, 'outputs')
 
 
 combined_output=os.path.join(outputs_path,  "combined.csv")

src/codes/import_biomass.py

@@ -1,8 +1,8 @@
-import os
+import sys,os
 import sys
-from tools import GeoserverClient
 import json
-import os
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))
+from codes.tools import GeoserverClient
 
 
 print ("")
@@ -14,13 +14,12 @@
 
 
 # Constants
-filepath = '../data.json'
+project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+filepath = os.path.join(project_root, 'data.json')
 
 with open(filepath) as f:
     data = json.load(f)
 
-all_path = data["base_directory"]
-
 geo_url = data["geoserve_url"]
 geo_user = data["geoserver_user"]
 geo_pwd = data["geoserver_pwd"]
@@ -29,8 +28,8 @@
 workspace_name = "waterpoints_et"
 forecast_type = "biomass"
 country_iso = workspace_name.split("_")[1]
-folder_data = '../data'
-outputs_path = '../outputs'
+folder_data = os.path.join(project_root, 'data')
+outputs_path = os.path.join(project_root, 'outputs')
 folder_layers = os.path.join(folder_data, "layers")
 folder_properties = os.path.join(folder_layers, forecast_type+"_properties")
 folder_tmp = os.path.join(folder_data, "tmp")

src/codes/master_script.py

@@ -17,6 +17,6 @@ def master_script(selected_files):
 
 if __name__ == "__main__":
     # specific files to execute
-    selected_files_to_execute = ["data_extraction.py", "rasterize.py","import_biomass.py",]
+    selected_files_to_execute = ["data_extraction.py","gwr_model.py", "rasterize.py","import_biomass.py",]
 
     master_script(selected_files_to_execute)

src/codes/rasterize.py

@@ -20,14 +20,9 @@
 
 
 # Constants
-filepath = '../data.json'
-
-with open(filepath) as f:
-    data = json.load(f)
-
-
-outputs_path = '../outputs'
-data_path='../data'
+project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+outputs_path = os.path.join(project_root, 'outputs')
+data_path = os.path.join(project_root, 'data')
 
 
 layers_path = os.path.join(data_path, "layers")