From 8216e4c66e760102c0c0d327cc22a1f21a43cba7 Mon Sep 17 00:00:00 2001 From: Shengting Cui Date: Fri, 8 Dec 2023 16:40:06 +0000 Subject: [PATCH 01/19] First draft workflow code --- .github/workflows/ngen_integration.yaml | 122 ++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 .github/workflows/ngen_integration.yaml diff --git a/.github/workflows/ngen_integration.yaml b/.github/workflows/ngen_integration.yaml new file mode 100644 index 0000000..6ba5cba --- /dev/null +++ b/.github/workflows/ngen_integration.yaml @@ -0,0 +1,122 @@ +# Test ngen-cfe integration + +name: Ngen Integration Tests + +# Controls when the action will run. +on: + push: + branches: [ main, dev, notreal ] + pull_request: + branches: [ main, dev, notreal ] + workflow_dispatch: + +env: + # Obtained from https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources + # TODO: add something later that can check the runners for hyperthreading + LINUX_NUM_PROC_CORES: 2 + MACOS_NUM_PROC_CORES: 3 + ASAN_OPTIONS: detect_leaks=false + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # Run general unit tests in linux environment + test_surfacebmi_plus_cfe: + # The type of runner that the job will run on + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + fail-fast: false + runs-on: ${{ matrix.os }} + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - name: checkout actions in another repo + uses: actions/checkout@v4 + with: + repository: noaa-owp/ngen + + - name: Build ISO C Fortran BMI + id: submod_build_2 + uses: ./.github/actions/ngen-submod-build + with: + mod-dir: "extern/iso_c_fortran_bmi/" + + - name: Build CFE + id: submod_build_3 + uses: ./.github/actions/ngen-submod-build + with: + mod-dir: "extern/cfe/" + targets: "cfebmi" + + - name: Build PET + id: submod_build_5 + uses: ./.github/actions/ngen-submod-build + with: + mod-dir: "extern/evapotranspiration/evapotranspiration" + targets: "petbmi" + + - name: Build SLoTH + id: submod_build_6 + uses: ./.github/actions/ngen-submod-build + with: + mod-dir: "extern/sloth/" + targets: "slothmodel" + + # Build Ngen and save some files + - name: Build Ngen + id: ngen_id1 + uses: ./.github/actions/ngen-build + with: + targets: "ngen" + build-cores: ${{ env.LINUX_NUM_PROC_CORES }} + #is this required for this test? + bmi_c: 'ON' + bmi_fortran: 'ON' + timeout-minutes: 15 + + - name: Cleanup Ngen Build + run: | + # Save ngen build artifacts to temp directory + mv ${{ steps.ngen_id1.outputs.build-dir }} ${{runner.temp}}/ngen-build/ + mv .github ${{runner.temp}}/.github + mv extern ${{runner.temp}}/extern + mv data ${{runner.temp}}/data + + # Checkout and build the topmodel + - name: Checkout the commit + uses: actions/checkout@v4 + + - name: Prepare to Build topmodel Library for Ngen + run: | + # Move files to appropriate directory to build topmodel library + mv ${{runner.temp}}/extern extern + mv -f ./* extern/topmodel/topmodel + mv ${{runner.temp}}/.github .github + + - name: Build Surfacebmi + id: submod_build_1 + uses: ./.github/actions/ngen-submod-build + with: + mod-dir: " extern/topmodel/" + targets: "surfacebmi" + + - name: Move files to appropriate directory to run ngen + run: | + #Not sure if all the steps are necessary, check later + mv ${{runner.temp}}/ngen-build ./ngen-build + #mv ${{runner.temp}}/extern/sloth extern + #mv ${{runner.temp}}/extern/iso_c_fortran_bmi extern + #cp -r ${{runner.temp}}/extern/evapotranspiration extern + mv ${{runner.temp}}/data data + + # Run ngen with topmodel with CFE + inputfile='data/example_bmi_multi_realization_config.json' + ./ngen-build/ngen ./data/catchment_data.geojson "cat-27" ./data/nexus_data.geojson "nex-26" $inputfile + + # The following remove a post ngen build error likely related to the temporary files left over during the + # build process + - name: Re-checkout Ngen + uses: actions/checkout@v3 + with: + repository: noaa-owp/ngen From 470f30111a65a7dcb8d961e332725f5e53dea6c5 Mon Sep 17 00:00:00 2001 From: Shengting Cui Date: Fri, 8 Dec 2023 17:10:16 +0000 Subject: [PATCH 02/19] Change main to master on push --- .github/workflows/ngen_integration.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ngen_integration.yaml b/.github/workflows/ngen_integration.yaml index 6ba5cba..ad1e70d 100644 --- a/.github/workflows/ngen_integration.yaml +++ b/.github/workflows/ngen_integration.yaml @@ -5,9 +5,9 @@ name: Ngen Integration Tests # Controls when the action will run. on: push: - branches: [ main, dev, notreal ] + branches: [ master, dev, notreal ] pull_request: - branches: [ main, dev, notreal ] + branches: [ master, dev, notreal ] workflow_dispatch: env: From 92ce2607107ebc0b3a1d38553e806d3ddf5786b9 Mon Sep 17 00:00:00 2001 From: Shengting Cui Date: Fri, 8 Dec 2023 17:42:51 +0000 Subject: [PATCH 03/19] Clean extern/topmodel/topmodel/ before mv --- .github/workflows/ngen_integration.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ngen_integration.yaml b/.github/workflows/ngen_integration.yaml index ad1e70d..46aebdb 100644 --- a/.github/workflows/ngen_integration.yaml +++ b/.github/workflows/ngen_integration.yaml @@ -91,7 +91,8 @@ jobs: run: | # Move files to appropriate directory to build topmodel library mv ${{runner.temp}}/extern extern - mv -f ./* extern/topmodel/topmodel + rm -rf extern/topmodel/topmodel/* + mv ./* extern/topmodel/topmodel mv ${{runner.temp}}/.github .github - name: Build Surfacebmi From 5ad884257ee52ea0c087cbacf977c9cf810c2863 Mon Sep 17 00:00:00 2001 From: Shengting Cui Date: Fri, 8 Dec 2023 21:04:22 +0000 Subject: [PATCH 04/19] Move files to appropriate place to build topmodel lib --- .github/workflows/ngen_integration.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ngen_integration.yaml b/.github/workflows/ngen_integration.yaml index 46aebdb..4ba8c94 100644 --- a/.github/workflows/ngen_integration.yaml +++ b/.github/workflows/ngen_integration.yaml @@ -90,9 +90,9 @@ jobs: - name: Prepare to Build topmodel Library for Ngen run: | # Move files to appropriate directory to build topmodel library + rm -rf ${{runner.temp}}/extern/topmodel/topmodel/* + mv ./* ${{runner.temp}}/extern/topmodel/topmodel mv ${{runner.temp}}/extern extern - rm -rf extern/topmodel/topmodel/* - mv ./* extern/topmodel/topmodel mv ${{runner.temp}}/.github .github - name: Build Surfacebmi From bd5aa3ef89dfb881866a53f1b740ae44a7edabbe Mon Sep 17 00:00:00 2001 From: Shengting Cui Date: Fri, 8 Dec 2023 21:31:39 +0000 Subject: [PATCH 05/19] Locate ngen-submod-build action.yaml --- .github/workflows/ngen_integration.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ngen_integration.yaml b/.github/workflows/ngen_integration.yaml index 4ba8c94..518bbeb 100644 --- a/.github/workflows/ngen_integration.yaml +++ b/.github/workflows/ngen_integration.yaml @@ -93,7 +93,10 @@ jobs: rm -rf ${{runner.temp}}/extern/topmodel/topmodel/* mv ./* ${{runner.temp}}/extern/topmodel/topmodel mv ${{runner.temp}}/extern extern - mv ${{runner.temp}}/.github .github + echo `pwd` + echo `ls -l ${{runner.temp}}/.github/actions/ngen-submod-build` + mv ${{runner.temp}}/.github ./.github + echo `ls -l ./.github/actions/ngen-submod-build` - name: Build Surfacebmi id: submod_build_1 From 497b9243d1100c22beb06ef3cbc20a0ffc308f94 Mon Sep 17 00:00:00 2001 From: Shengting Cui Date: Fri, 8 Dec 2023 22:49:45 +0000 Subject: [PATCH 06/19] Test GITHUB_OUTPUT --- .github/workflows/ngen_integration.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ngen_integration.yaml b/.github/workflows/ngen_integration.yaml index 518bbeb..79dde8f 100644 --- a/.github/workflows/ngen_integration.yaml +++ b/.github/workflows/ngen_integration.yaml @@ -92,11 +92,11 @@ jobs: # Move files to appropriate directory to build topmodel library rm -rf ${{runner.temp}}/extern/topmodel/topmodel/* mv ./* ${{runner.temp}}/extern/topmodel/topmodel + # This line of code will override the new topmodel commit, need change mv ${{runner.temp}}/extern extern - echo `pwd` - echo `ls -l ${{runner.temp}}/.github/actions/ngen-submod-build` + echo "runner.temp-dir=$(echo ${{ runner.temp }})" >> $GITHUB_OUTPUT mv ${{runner.temp}}/.github ./.github - echo `ls -l ./.github/actions/ngen-submod-build` + echo "runner.temp-github-dir=$(echo ${{ runner.temp }}/.github)" >> $GITHUB_OUTPUT - name: Build Surfacebmi id: submod_build_1 From 0b90dda20dac9e70bcfcd84612a720ff5fd20fa0 Mon Sep 17 00:00:00 2001 From: Shengting Cui Date: Mon, 11 Dec 2023 14:54:19 +0000 Subject: [PATCH 07/19] Try a new version --- .github/workflows/ngen_integration.yaml | 40 ++++++++----------------- 1 file changed, 13 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ngen_integration.yaml b/.github/workflows/ngen_integration.yaml index 79dde8f..a1bd61c 100644 --- a/.github/workflows/ngen_integration.yaml +++ b/.github/workflows/ngen_integration.yaml @@ -30,6 +30,16 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: + # Checkout and build the topmodel + - name: Checkout the commit + uses: actions/checkout@v4 + + - name: Save Topmodel to a Temp Directory + run: | + # Move files to appropriate temporary directory + mkdir ${{runner.temp}}/topmodel + mv ./* ${{runner.temp}}/topmodel + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - name: checkout actions in another repo uses: actions/checkout@v4 @@ -75,28 +85,11 @@ jobs: bmi_fortran: 'ON' timeout-minutes: 15 - - name: Cleanup Ngen Build - run: | - # Save ngen build artifacts to temp directory - mv ${{ steps.ngen_id1.outputs.build-dir }} ${{runner.temp}}/ngen-build/ - mv .github ${{runner.temp}}/.github - mv extern ${{runner.temp}}/extern - mv data ${{runner.temp}}/data - - # Checkout and build the topmodel - - name: Checkout the commit - uses: actions/checkout@v4 - - name: Prepare to Build topmodel Library for Ngen run: | # Move files to appropriate directory to build topmodel library - rm -rf ${{runner.temp}}/extern/topmodel/topmodel/* - mv ./* ${{runner.temp}}/extern/topmodel/topmodel - # This line of code will override the new topmodel commit, need change - mv ${{runner.temp}}/extern extern - echo "runner.temp-dir=$(echo ${{ runner.temp }})" >> $GITHUB_OUTPUT - mv ${{runner.temp}}/.github ./.github - echo "runner.temp-github-dir=$(echo ${{ runner.temp }}/.github)" >> $GITHUB_OUTPUT + rm -rf extern/topmodel/topmodel/* + mv ${{runner.temp}}/topmodel/* extern/topmodel/topmodel - name: Build Surfacebmi id: submod_build_1 @@ -105,15 +98,8 @@ jobs: mod-dir: " extern/topmodel/" targets: "surfacebmi" - - name: Move files to appropriate directory to run ngen + - name: Run Ngen Test run: | - #Not sure if all the steps are necessary, check later - mv ${{runner.temp}}/ngen-build ./ngen-build - #mv ${{runner.temp}}/extern/sloth extern - #mv ${{runner.temp}}/extern/iso_c_fortran_bmi extern - #cp -r ${{runner.temp}}/extern/evapotranspiration extern - mv ${{runner.temp}}/data data - # Run ngen with topmodel with CFE inputfile='data/example_bmi_multi_realization_config.json' ./ngen-build/ngen ./data/catchment_data.geojson "cat-27" ./data/nexus_data.geojson "nex-26" $inputfile From 88fbf3a1d2afe2dfea3be9c50b7a62a2682d41a6 Mon Sep 17 00:00:00 2001 From: Shengting Cui Date: Mon, 11 Dec 2023 15:16:20 +0000 Subject: [PATCH 08/19] Fix a bug --- .github/workflows/ngen_integration.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ngen_integration.yaml b/.github/workflows/ngen_integration.yaml index a1bd61c..713557a 100644 --- a/.github/workflows/ngen_integration.yaml +++ b/.github/workflows/ngen_integration.yaml @@ -91,12 +91,12 @@ jobs: rm -rf extern/topmodel/topmodel/* mv ${{runner.temp}}/topmodel/* extern/topmodel/topmodel - - name: Build Surfacebmi + - name: Build Topmodel id: submod_build_1 uses: ./.github/actions/ngen-submod-build with: mod-dir: " extern/topmodel/" - targets: "surfacebmi" + targets: "topmodelbmi" - name: Run Ngen Test run: | From e5b4f0b0d556296c03e4ae4fffb45a82190539b2 Mon Sep 17 00:00:00 2001 From: Shengting Cui Date: Mon, 11 Dec 2023 15:40:10 +0000 Subject: [PATCH 09/19] Move Ngen build directory to working directory --- .github/workflows/ngen_integration.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ngen_integration.yaml b/.github/workflows/ngen_integration.yaml index 713557a..8354023 100644 --- a/.github/workflows/ngen_integration.yaml +++ b/.github/workflows/ngen_integration.yaml @@ -101,6 +101,7 @@ jobs: - name: Run Ngen Test run: | # Run ngen with topmodel with CFE + mv ${{ steps.ngen_id1.outputs.build-dir }} ./ngen-build/ inputfile='data/example_bmi_multi_realization_config.json' ./ngen-build/ngen ./data/catchment_data.geojson "cat-27" ./data/nexus_data.geojson "nex-26" $inputfile From 3d0f4617409408bb6467cb35e8c9ae753e167959 Mon Sep 17 00:00:00 2001 From: Shengting Cui Date: Mon, 11 Dec 2023 16:00:19 +0000 Subject: [PATCH 10/19] Add build noah-owp-modular library --- .github/workflows/ngen_integration.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ngen_integration.yaml b/.github/workflows/ngen_integration.yaml index 8354023..da7891f 100644 --- a/.github/workflows/ngen_integration.yaml +++ b/.github/workflows/ngen_integration.yaml @@ -46,6 +46,13 @@ jobs: with: repository: noaa-owp/ngen + - name: Build Surfacebmi + id: submod_build_1 + uses: ./.github/actions/ngen-submod-build + with: + mod-dir: " extern/noah-owp-modular/" + targets: "surfacebmi" + - name: Build ISO C Fortran BMI id: submod_build_2 uses: ./.github/actions/ngen-submod-build @@ -92,7 +99,7 @@ jobs: mv ${{runner.temp}}/topmodel/* extern/topmodel/topmodel - name: Build Topmodel - id: submod_build_1 + id: submod_build_4 uses: ./.github/actions/ngen-submod-build with: mod-dir: " extern/topmodel/" From 3351ef60618f7a8b16c3d7fac7cef50f93b86779 Mon Sep 17 00:00:00 2001 From: Shengting Cui Date: Tue, 12 Dec 2023 15:15:49 +0000 Subject: [PATCH 11/19] Add another run ngen test --- .github/workflows/ngen_integration.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ngen_integration.yaml b/.github/workflows/ngen_integration.yaml index da7891f..380f7f8 100644 --- a/.github/workflows/ngen_integration.yaml +++ b/.github/workflows/ngen_integration.yaml @@ -112,6 +112,11 @@ jobs: inputfile='data/example_bmi_multi_realization_config.json' ./ngen-build/ngen ./data/catchment_data.geojson "cat-27" ./data/nexus_data.geojson "nex-26" $inputfile + - name: Run surfacebmi, cfebmi and petbmi + run: | + inputfile='data/example_bmi_multi_realization_config_w_noah_pet_cfe.json' + ./cmake_build/ngen data/catchment_data.geojson "cat-27" data/nexus_data.geojson "nex-26" $inputfile + # The following remove a post ngen build error likely related to the temporary files left over during the # build process - name: Re-checkout Ngen From 25958ce211b1ebcece65e32b730d6da7634eef07 Mon Sep 17 00:00:00 2001 From: Shengting Cui Date: Tue, 12 Dec 2023 16:00:35 +0000 Subject: [PATCH 12/19] Fix a bug --- .github/workflows/ngen_integration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ngen_integration.yaml b/.github/workflows/ngen_integration.yaml index 380f7f8..a4086cc 100644 --- a/.github/workflows/ngen_integration.yaml +++ b/.github/workflows/ngen_integration.yaml @@ -115,7 +115,7 @@ jobs: - name: Run surfacebmi, cfebmi and petbmi run: | inputfile='data/example_bmi_multi_realization_config_w_noah_pet_cfe.json' - ./cmake_build/ngen data/catchment_data.geojson "cat-27" data/nexus_data.geojson "nex-26" $inputfile + ./ngen_build/ngen ./data/catchment_data.geojson "cat-27" ./data/nexus_data.geojson "nex-26" $inputfile # The following remove a post ngen build error likely related to the temporary files left over during the # build process From d8baa045629706b51bac95e8078354293e767148 Mon Sep 17 00:00:00 2001 From: Shengting Cui Date: Tue, 12 Dec 2023 16:41:51 +0000 Subject: [PATCH 13/19] Fix one more bug --- .github/workflows/ngen_integration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ngen_integration.yaml b/.github/workflows/ngen_integration.yaml index a4086cc..dcef4e2 100644 --- a/.github/workflows/ngen_integration.yaml +++ b/.github/workflows/ngen_integration.yaml @@ -115,7 +115,7 @@ jobs: - name: Run surfacebmi, cfebmi and petbmi run: | inputfile='data/example_bmi_multi_realization_config_w_noah_pet_cfe.json' - ./ngen_build/ngen ./data/catchment_data.geojson "cat-27" ./data/nexus_data.geojson "nex-26" $inputfile + ./ngen-build/ngen ./data/catchment_data.geojson "cat-27" ./data/nexus_data.geojson "nex-26" $inputfile # The following remove a post ngen build error likely related to the temporary files left over during the # build process From 6101462cd7340cb1561f9b8d08f72340b1967ac3 Mon Sep 17 00:00:00 2001 From: Shengting Cui Date: Mon, 18 Dec 2023 14:40:28 +0000 Subject: [PATCH 14/19] Remove re-checkout ngen in post testing --- .github/workflows/build_and_run_unit_test.yml | 2 +- .github/workflows/ngen_integration.yaml | 13 +++---------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build_and_run_unit_test.yml b/.github/workflows/build_and_run_unit_test.yml index 04fccb5..a14cb07 100644 --- a/.github/workflows/build_and_run_unit_test.yml +++ b/.github/workflows/build_and_run_unit_test.yml @@ -30,7 +30,7 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: - name: Checkout the commit - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Build and Run Unit Test run: | diff --git a/.github/workflows/ngen_integration.yaml b/.github/workflows/ngen_integration.yaml index dcef4e2..c9e38d5 100644 --- a/.github/workflows/ngen_integration.yaml +++ b/.github/workflows/ngen_integration.yaml @@ -30,13 +30,13 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: - # Checkout and build the topmodel + # Checkout and save the topmodel to a temporary directory - name: Checkout the commit uses: actions/checkout@v4 - name: Save Topmodel to a Temp Directory run: | - # Move files to appropriate temporary directory + # Move files to a temporary directory mkdir ${{runner.temp}}/topmodel mv ./* ${{runner.temp}}/topmodel @@ -92,7 +92,7 @@ jobs: bmi_fortran: 'ON' timeout-minutes: 15 - - name: Prepare to Build topmodel Library for Ngen + - name: Prepare to Build Topmodel Library for Ngen run: | # Move files to appropriate directory to build topmodel library rm -rf extern/topmodel/topmodel/* @@ -116,10 +116,3 @@ jobs: run: | inputfile='data/example_bmi_multi_realization_config_w_noah_pet_cfe.json' ./ngen-build/ngen ./data/catchment_data.geojson "cat-27" ./data/nexus_data.geojson "nex-26" $inputfile - - # The following remove a post ngen build error likely related to the temporary files left over during the - # build process - - name: Re-checkout Ngen - uses: actions/checkout@v3 - with: - repository: noaa-owp/ngen From 6e2493c8d24d48bc5b962af8bb46d246ac9ee690 Mon Sep 17 00:00:00 2001 From: Shengting Cui Date: Tue, 19 Dec 2023 21:52:11 +0000 Subject: [PATCH 15/19] Add new test for topmodel --- .github/workflows/ngen_integration.yaml | 4 +- data/realization_config_topmodel_ngenCI.json | 84 ++++++++++++++++++++ 2 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 data/realization_config_topmodel_ngenCI.json diff --git a/.github/workflows/ngen_integration.yaml b/.github/workflows/ngen_integration.yaml index c9e38d5..b5caf8d 100644 --- a/.github/workflows/ngen_integration.yaml +++ b/.github/workflows/ngen_integration.yaml @@ -112,7 +112,7 @@ jobs: inputfile='data/example_bmi_multi_realization_config.json' ./ngen-build/ngen ./data/catchment_data.geojson "cat-27" ./data/nexus_data.geojson "nex-26" $inputfile - - name: Run surfacebmi, cfebmi and petbmi + - name: Run surfacebmi, topmodelbmi run: | - inputfile='data/example_bmi_multi_realization_config_w_noah_pet_cfe.json' + inputfile='extern/topmodel/topmodel/data/realization_config_topmodel_ngenCI.json' ./ngen-build/ngen ./data/catchment_data.geojson "cat-27" ./data/nexus_data.geojson "nex-26" $inputfile diff --git a/data/realization_config_topmodel_ngenCI.json b/data/realization_config_topmodel_ngenCI.json new file mode 100644 index 0000000..679830d --- /dev/null +++ b/data/realization_config_topmodel_ngenCI.json @@ -0,0 +1,84 @@ +{ + "global": { + "formulations": [ + { + "name": "bmi_multi", + "params": { + "model_type_name": "bmi_multi_noahowp_cfe", + "forcing_file": "", + "init_config": "", + "allow_exceed_end_time": true, + "main_output_variable": "Q_OUT", + "modules": [ + { + "name": "bmi_c++", + "params": { + "model_type_name": "bmi_c++_sloth", + "library_file": "./extern/sloth/cmake_build/libslothmodel", + "init_config": "/dev/null", + "allow_exceed_end_time": true, + "main_output_variable": "z", + "uses_forcing_file": false, + "model_params": { + "sloth_ice_fraction_schaake(1,double,m,node)": 0.0, + "sloth_ice_fraction_xinan(1,double,1,node)": 0.0, + "sloth_smp(1,double,1,node)": 0.0 + } + } + }, + { + "name": "bmi_fortran", + "params": { + "model_type_name": "bmi_fortran_noahowp", + "library_file": "./extern/noah-owp-modular/cmake_build/libsurfacebmi", + "forcing_file": "", + "init_config": "./data/bmi/fortran/noah-owp-modular-init-{{id}}.namelist.input", + "allow_exceed_end_time": true, + "main_output_variable": "QINSUR", + "variables_names_map": { + "PRCPNONC": "atmosphere_water__liquid_equivalent_precipitation_rate", + "Q2": "atmosphere_air_water~vapor__relative_saturation", + "SFCTMP": "land_surface_air__temperature", + "UU": "land_surface_wind__x_component_of_velocity", + "VV": "land_surface_wind__y_component_of_velocity", + "LWDN": "land_surface_radiation~incoming~longwave__energy_flux", + "SOLDN": "land_surface_radiation~incoming~shortwave__energy_flux", + "SFCPRS": "land_surface_air__pressure" + }, + "uses_forcing_file": false + } + }, + { + "name": "bmi_c", + "params": { + "model_type_name": "bmi_c_topmodel", + "library_file": "./extern/topmodel/cmake_build/libtopmodelbmi.so", + "forcing_file": "", + "init_config": "./extern/topmodel/topmodel/data/topmod.run", + "allow_exceed_end_time": true, + "main_output_variable": "Qout", + "registration_function": "register_bmi_topmodel", + "variables_names_map": { + "water_potential_evaporation_flux": "EVAPOTRANS", + "atmosphere_water__liquid_equivalent_precipitation_rate": "QINSUR" + }, + "uses_forcing_file": false + } + } + ], + "uses_forcing_file": false + } + } + ], + "forcing": { + "file_pattern": ".*{{id}}.*..csv", + "path": "./data/forcing/", + "provider": "CsvPerFeature" + } + }, + "time": { + "start_time": "2015-12-01 00:00:00", + "end_time": "2015-12-30 23:00:00", + "output_interval": 3600 + } +} From a0366719ab9cc950ec77eade231dc682743a6d1b Mon Sep 17 00:00:00 2001 From: Shengting Cui Date: Tue, 19 Dec 2023 22:09:16 +0000 Subject: [PATCH 16/19] cp topmodel data into ngen data --- .github/workflows/ngen_integration.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ngen_integration.yaml b/.github/workflows/ngen_integration.yaml index b5caf8d..551dc16 100644 --- a/.github/workflows/ngen_integration.yaml +++ b/.github/workflows/ngen_integration.yaml @@ -114,5 +114,6 @@ jobs: - name: Run surfacebmi, topmodelbmi run: | + cp extern/topmodel/topmodel/data/*.dat data inputfile='extern/topmodel/topmodel/data/realization_config_topmodel_ngenCI.json' ./ngen-build/ngen ./data/catchment_data.geojson "cat-27" ./data/nexus_data.geojson "nex-26" $inputfile From 25989b1a48ce675bf7e4610bde08fcf27c677ca9 Mon Sep 17 00:00:00 2001 From: Shengting Cui Date: Tue, 19 Dec 2023 22:29:06 +0000 Subject: [PATCH 17/19] Rename Q_OUT --- data/realization_config_topmodel_ngenCI.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/realization_config_topmodel_ngenCI.json b/data/realization_config_topmodel_ngenCI.json index 679830d..a072dea 100644 --- a/data/realization_config_topmodel_ngenCI.json +++ b/data/realization_config_topmodel_ngenCI.json @@ -8,7 +8,7 @@ "forcing_file": "", "init_config": "", "allow_exceed_end_time": true, - "main_output_variable": "Q_OUT", + "main_output_variable": "QINSUR", "modules": [ { "name": "bmi_c++", @@ -56,7 +56,7 @@ "forcing_file": "", "init_config": "./extern/topmodel/topmodel/data/topmod.run", "allow_exceed_end_time": true, - "main_output_variable": "Qout", + "main_output_variable": "QINSUR", "registration_function": "register_bmi_topmodel", "variables_names_map": { "water_potential_evaporation_flux": "EVAPOTRANS", From aa2c38ad2f5a167e259c32867d5a32262733ce17 Mon Sep 17 00:00:00 2001 From: Shengting Cui Date: Tue, 19 Dec 2023 23:10:57 +0000 Subject: [PATCH 18/19] Rename Qout --- data/realization_config_topmodel_ngenCI.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/realization_config_topmodel_ngenCI.json b/data/realization_config_topmodel_ngenCI.json index a072dea..fe6106e 100644 --- a/data/realization_config_topmodel_ngenCI.json +++ b/data/realization_config_topmodel_ngenCI.json @@ -8,7 +8,7 @@ "forcing_file": "", "init_config": "", "allow_exceed_end_time": true, - "main_output_variable": "QINSUR", + "main_output_variable": "Qout", "modules": [ { "name": "bmi_c++", @@ -56,7 +56,7 @@ "forcing_file": "", "init_config": "./extern/topmodel/topmodel/data/topmod.run", "allow_exceed_end_time": true, - "main_output_variable": "QINSUR", + "main_output_variable": "Qout", "registration_function": "register_bmi_topmodel", "variables_names_map": { "water_potential_evaporation_flux": "EVAPOTRANS", From 37c5044715c671e8f03f6e3d5d9d6a073ef341fc Mon Sep 17 00:00:00 2001 From: JessicaGarrett-NOAA <30940444+madMatchstick@users.noreply.github.com> Date: Thu, 28 Dec 2023 14:11:44 -0700 Subject: [PATCH 19/19] Add pet-topmodel ngen run to workflow (#1) * testing ngen workflow with tmod-pet * pet config is format ini * pet lib path fix * move topmod data dir? * tmod var map adjust * re-add surfacebmi+topmod run --------- Co-authored-by: Jessica Garrett --- .github/workflows/ngen_integration.yaml | 20 +++---- .../realization_config_topmodel_pet_ngen.json | 58 +++++++++++++++++++ 2 files changed, 65 insertions(+), 13 deletions(-) create mode 100644 data/realization_config_topmodel_pet_ngen.json diff --git a/.github/workflows/ngen_integration.yaml b/.github/workflows/ngen_integration.yaml index 551dc16..15e354d 100644 --- a/.github/workflows/ngen_integration.yaml +++ b/.github/workflows/ngen_integration.yaml @@ -20,7 +20,7 @@ env: # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: # Run general unit tests in linux environment - test_surfacebmi_plus_cfe: + test_topmodelbmi: # The type of runner that the job will run on strategy: matrix: @@ -59,13 +59,6 @@ jobs: with: mod-dir: "extern/iso_c_fortran_bmi/" - - name: Build CFE - id: submod_build_3 - uses: ./.github/actions/ngen-submod-build - with: - mod-dir: "extern/cfe/" - targets: "cfebmi" - - name: Build PET id: submod_build_5 uses: ./.github/actions/ngen-submod-build @@ -104,16 +97,17 @@ jobs: with: mod-dir: " extern/topmodel/" targets: "topmodelbmi" - - - name: Run Ngen Test + + - name: Run petbmi, topmodelbmi run: | - # Run ngen with topmodel with CFE + # Run ngen with topmodel & pet mv ${{ steps.ngen_id1.outputs.build-dir }} ./ngen-build/ - inputfile='data/example_bmi_multi_realization_config.json' + cp extern/topmodel/topmodel/data/*.dat data + inputfile='extern/topmodel/topmodel/data/realization_config_topmodel_pet_ngen.json' ./ngen-build/ngen ./data/catchment_data.geojson "cat-27" ./data/nexus_data.geojson "nex-26" $inputfile - name: Run surfacebmi, topmodelbmi run: | - cp extern/topmodel/topmodel/data/*.dat data + # Run ngen with noahowpmod & topmodel inputfile='extern/topmodel/topmodel/data/realization_config_topmodel_ngenCI.json' ./ngen-build/ngen ./data/catchment_data.geojson "cat-27" ./data/nexus_data.geojson "nex-26" $inputfile diff --git a/data/realization_config_topmodel_pet_ngen.json b/data/realization_config_topmodel_pet_ngen.json new file mode 100644 index 0000000..239e8fa --- /dev/null +++ b/data/realization_config_topmodel_pet_ngen.json @@ -0,0 +1,58 @@ +{ + "global": { + "formulations": [ + { + "name": "bmi_multi", + "params": { + "model_type_name": "bmi_multi_pet_topmodel", + "forcing_file": "", + "init_config": "", + "allow_exceed_end_time": true, + "main_output_variable": "Qout", + "modules": [ + { + "name": "bmi_c", + "params": { + "model_type_name": "bmi_c_pet", + "library_file": "./extern/evapotranspiration/evapotranspiration/cmake_build/libpetbmi", + "forcing_file": "", + "init_config": "./data/bmi/c/pet/{{id}}_bmi_config.ini", + "allow_exceed_end_time": true, + "main_output_variable": "water_potential_evaporation_flux", + "registration_function":"register_bmi_pet", + "uses_forcing_file": false + } + }, + { + "name": "bmi_c", + "params": { + "model_type_name": "bmi_c_topmodel", + "library_file": "./extern/topmodel/cmake_build/libtopmodelbmi.so", + "forcing_file": "", + "init_config": "./extern/topmodel/topmodel/data/topmod.run", + "allow_exceed_end_time": true, + "main_output_variable": "Qout", + "registration_function": "register_bmi_topmodel", + "variables_names_map": { + "water_potential_evaporation_flux": "water_potential_evaporation_flux" + }, + "uses_forcing_file": false + } + } + ], + "uses_forcing_file": false + } + } + ], + "forcing": { + "file_pattern": ".*{{id}}.*..csv", + "path": "./data/forcing/", + "provider": "CsvPerFeature" + } + }, + "time": { + "start_time": "2015-12-01 00:00:00", + "end_time": "2015-12-30 23:00:00", + "output_interval": 3600 + } +}