Skip to content

Commit

Permalink
Workflow code that run topmodel within ngen Framework (#42)
Browse files Browse the repository at this point in the history 
* First draft workflow code

* Change main to master on push

* Clean extern/topmodel/topmodel/ before mv

* Move files to appropriate place to build topmodel lib

* Locate ngen-submod-build action.yaml

* Test GITHUB_OUTPUT

* Try a new version

* Fix a bug

* Move Ngen build directory to working directory

* Add build noah-owp-modular library

* Add another run ngen test

* Fix a bug

* Fix one more bug

* Remove re-checkout ngen in post testing

* Add new test for topmodel

* cp topmodel data into ngen data

* Rename Q_OUT

* Rename Qout

* Add pet-topmodel ngen run to workflow (#1)

* testing ngen workflow with tmod-pet

* pet config is format ini

* pet lib path fix

* move topmod data dir?

* tmod var map adjust

* re-add surfacebmi+topmod run

---------

Co-authored-by: Jessica Garrett <jessicagarrett@Lynkers-MacBook-Pro.local>

---------

Co-authored-by: JessicaGarrett-NOAA <30940444+madMatchstick@users.noreply.github.com>
Co-authored-by: Jessica Garrett <jessicagarrett@Lynkers-MacBook-Pro.local>
  • Loading branch information
@stcui007 @madMatchstick
3 people authored Dec 29, 2023
1 parent 03bb233 commit 909942b
Show file tree
Hide file tree
Showing 4 changed files with 256 additions and 1 deletion.
2 changes: 1 addition & 1 deletion .github/workflows/build_and_run_unit_test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
- name: Checkout the commit
uses: actions/checkout@v2
uses: actions/checkout@v4

- name: Build and Run Unit Test
run: |
Expand Down
113 changes: 113 additions & 0 deletions .github/workflows/ngen_integration.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
# Test ngen-cfe integration

name: Ngen Integration Tests

# Controls when the action will run.
on:
push:
branches: [ master, dev, notreal ]
pull_request:
branches: [ master, dev, notreal ]
workflow_dispatch:

env:
# Obtained from https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources
# TODO: add something later that can check the runners for hyperthreading
LINUX_NUM_PROC_CORES: 2
MACOS_NUM_PROC_CORES: 3
ASAN_OPTIONS: detect_leaks=false

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
# Run general unit tests in linux environment
test_topmodelbmi:
# The type of runner that the job will run on
strategy:
matrix:
os: [ubuntu-latest, macos-latest]
fail-fast: false
runs-on: ${{ matrix.os }}

# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checkout and save the topmodel to a temporary directory
- name: Checkout the commit
uses: actions/checkout@v4

- name: Save Topmodel to a Temp Directory
run: |
# Move files to a temporary directory
mkdir ${{runner.temp}}/topmodel
mv ./* ${{runner.temp}}/topmodel
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- name: checkout actions in another repo
uses: actions/checkout@v4
with:
repository: noaa-owp/ngen

- name: Build Surfacebmi
id: submod_build_1
uses: ./.github/actions/ngen-submod-build
with:
mod-dir: " extern/noah-owp-modular/"
targets: "surfacebmi"

- name: Build ISO C Fortran BMI
id: submod_build_2
uses: ./.github/actions/ngen-submod-build
with:
mod-dir: "extern/iso_c_fortran_bmi/"

- name: Build PET
id: submod_build_5
uses: ./.github/actions/ngen-submod-build
with:
mod-dir: "extern/evapotranspiration/evapotranspiration"
targets: "petbmi"

- name: Build SLoTH
id: submod_build_6
uses: ./.github/actions/ngen-submod-build
with:
mod-dir: "extern/sloth/"
targets: "slothmodel"

# Build Ngen and save some files
- name: Build Ngen
id: ngen_id1
uses: ./.github/actions/ngen-build
with:
targets: "ngen"
build-cores: ${{ env.LINUX_NUM_PROC_CORES }}
#is this required for this test?
bmi_c: 'ON'
bmi_fortran: 'ON'
timeout-minutes: 15

- name: Prepare to Build Topmodel Library for Ngen
run: |
# Move files to appropriate directory to build topmodel library
rm -rf extern/topmodel/topmodel/*
mv ${{runner.temp}}/topmodel/* extern/topmodel/topmodel
- name: Build Topmodel
id: submod_build_4
uses: ./.github/actions/ngen-submod-build
with:
mod-dir: " extern/topmodel/"
targets: "topmodelbmi"

- name: Run petbmi, topmodelbmi
run: |
# Run ngen with topmodel & pet
mv ${{ steps.ngen_id1.outputs.build-dir }} ./ngen-build/
cp extern/topmodel/topmodel/data/*.dat data
inputfile='extern/topmodel/topmodel/data/realization_config_topmodel_pet_ngen.json'
./ngen-build/ngen ./data/catchment_data.geojson "cat-27" ./data/nexus_data.geojson "nex-26" $inputfile
- name: Run surfacebmi, topmodelbmi
run: |
# Run ngen with noahowpmod & topmodel
inputfile='extern/topmodel/topmodel/data/realization_config_topmodel_ngenCI.json'
./ngen-build/ngen ./data/catchment_data.geojson "cat-27" ./data/nexus_data.geojson "nex-26" $inputfile
84 changes: 84 additions & 0 deletions data/realization_config_topmodel_ngenCI.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
{
"global": {
"formulations": [
{
"name": "bmi_multi",
"params": {
"model_type_name": "bmi_multi_noahowp_cfe",
"forcing_file": "",
"init_config": "",
"allow_exceed_end_time": true,
"main_output_variable": "Qout",
"modules": [
{
"name": "bmi_c++",
"params": {
"model_type_name": "bmi_c++_sloth",
"library_file": "./extern/sloth/cmake_build/libslothmodel",
"init_config": "/dev/null",
"allow_exceed_end_time": true,
"main_output_variable": "z",
"uses_forcing_file": false,
"model_params": {
"sloth_ice_fraction_schaake(1,double,m,node)": 0.0,
"sloth_ice_fraction_xinan(1,double,1,node)": 0.0,
"sloth_smp(1,double,1,node)": 0.0
}
}
},
{
"name": "bmi_fortran",
"params": {
"model_type_name": "bmi_fortran_noahowp",
"library_file": "./extern/noah-owp-modular/cmake_build/libsurfacebmi",
"forcing_file": "",
"init_config": "./data/bmi/fortran/noah-owp-modular-init-{{id}}.namelist.input",
"allow_exceed_end_time": true,
"main_output_variable": "QINSUR",
"variables_names_map": {
"PRCPNONC": "atmosphere_water__liquid_equivalent_precipitation_rate",
"Q2": "atmosphere_air_water~vapor__relative_saturation",
"SFCTMP": "land_surface_air__temperature",
"UU": "land_surface_wind__x_component_of_velocity",
"VV": "land_surface_wind__y_component_of_velocity",
"LWDN": "land_surface_radiation~incoming~longwave__energy_flux",
"SOLDN": "land_surface_radiation~incoming~shortwave__energy_flux",
"SFCPRS": "land_surface_air__pressure"
},
"uses_forcing_file": false
}
},
{
"name": "bmi_c",
"params": {
"model_type_name": "bmi_c_topmodel",
"library_file": "./extern/topmodel/cmake_build/libtopmodelbmi.so",
"forcing_file": "",
"init_config": "./extern/topmodel/topmodel/data/topmod.run",
"allow_exceed_end_time": true,
"main_output_variable": "Qout",
"registration_function": "register_bmi_topmodel",
"variables_names_map": {
"water_potential_evaporation_flux": "EVAPOTRANS",
"atmosphere_water__liquid_equivalent_precipitation_rate": "QINSUR"
},
"uses_forcing_file": false
}
}
],
"uses_forcing_file": false
}
}
],
"forcing": {
"file_pattern": ".*{{id}}.*..csv",
"path": "./data/forcing/",
"provider": "CsvPerFeature"
}
},
"time": {
"start_time": "2015-12-01 00:00:00",
"end_time": "2015-12-30 23:00:00",
"output_interval": 3600
}
}
58 changes: 58 additions & 0 deletions data/realization_config_topmodel_pet_ngen.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
{
"global": {
"formulations": [
{
"name": "bmi_multi",
"params": {
"model_type_name": "bmi_multi_pet_topmodel",
"forcing_file": "",
"init_config": "",
"allow_exceed_end_time": true,
"main_output_variable": "Qout",
"modules": [
{
"name": "bmi_c",
"params": {
"model_type_name": "bmi_c_pet",
"library_file": "./extern/evapotranspiration/evapotranspiration/cmake_build/libpetbmi",
"forcing_file": "",
"init_config": "./data/bmi/c/pet/{{id}}_bmi_config.ini",
"allow_exceed_end_time": true,
"main_output_variable": "water_potential_evaporation_flux",
"registration_function":"register_bmi_pet",
"uses_forcing_file": false
}
},
{
"name": "bmi_c",
"params": {
"model_type_name": "bmi_c_topmodel",
"library_file": "./extern/topmodel/cmake_build/libtopmodelbmi.so",
"forcing_file": "",
"init_config": "./extern/topmodel/topmodel/data/topmod.run",
"allow_exceed_end_time": true,
"main_output_variable": "Qout",
"registration_function": "register_bmi_topmodel",
"variables_names_map": {
"water_potential_evaporation_flux": "water_potential_evaporation_flux"
},
"uses_forcing_file": false
}
}
],
"uses_forcing_file": false
}
}
],
"forcing": {
"file_pattern": ".*{{id}}.*..csv",
"path": "./data/forcing/",
"provider": "CsvPerFeature"
}
},
"time": {
"start_time": "2015-12-01 00:00:00",
"end_time": "2015-12-30 23:00:00",
"output_interval": 3600
}
}

0 comments on commit 909942b

Please sign in to comment.