diff --git a/.github/actions/show-workflow-trigger/README.md b/.github/actions/show-workflow-trigger/README.md new file mode 100644 index 000000000000..035c90d3288c --- /dev/null +++ b/.github/actions/show-workflow-trigger/README.md @@ -0,0 +1,3 @@ +# Composite action to show the trigger of a workflow + +If possible, prints also the user that triggered it diff --git a/.github/actions/show-workflow-trigger/action.yml b/.github/actions/show-workflow-trigger/action.yml new file mode 100644 index 000000000000..652e4999b0ea --- /dev/null +++ b/.github/actions/show-workflow-trigger/action.yml @@ -0,0 +1,27 @@ +name: 'Show workflow trigger' +description: 'Prints what triggered this workflow' + +runs: + using: "composite" + steps: + - name: Print trigger info + uses: actions/github-script@v7 + with: + script: | + const eventName = context.eventName; + const actor = context.actor || 'unknown'; // Default to 'unknown' if actor is not defined + let eventAction = 'N/A'; + + // Determine the event action based on the event type + if (eventName === 'pull_request') { + eventAction = context.payload.action || 'N/A'; + } else if (eventName === 'pull_request_review') { + eventAction = context.payload.review.state || 'N/A'; + } else if (eventName === 'workflow_dispatch') { + eventAction = 'manual trigger'; + } else if (eventName === 'schedule') { + eventAction = 'scheduled trigger'; + } + console.log(`The job was triggered by a ${eventName} event.`); + console.log(` - Event action: ${eventAction}`); + console.log(` - Triggered by: ${actor}`); diff --git a/.github/actions/test-all-scream/README.md b/.github/actions/test-all-scream/README.md new file mode 100644 index 000000000000..03ae55b4623f --- /dev/null +++ b/.github/actions/test-all-scream/README.md @@ -0,0 +1,33 @@ +# Composite action to call test-all-scream inside a workflow + +This action is meant to be used inside a workflow. E.g., + +```yaml +jobs: + my-testing: + steps: + ... + - name: run-test-all-scream + uses: ./.github/actions/eamxx-test-all-scream + with: + build_type: + machine: + run_type: +``` + +The input run_type is the only input that this action has to explicitly handle. +As such, this action checks that its value is one of the following. + +- nightly: runs tests and then submit to cdash +- at-run: runs tests without submitting to cdash +- bless: runs tests and copy output files to baselines folder + +As for build_type and machine, we do not prescribe a list of +valid values, as that will be handled by components/eamxx/scripts/test-all-scream. +If their values are not supported, it is up to test-all-scram to handle the error. +As a guideline, however, you may have to ensure that the following exist: + +- the file components/eamxx/cmake/machine-files/${machine}.cmake +- the entry ${machine} in the MACHINE_METADATA dict in components/eamxx/scripts/machine_specs.py + + Questions? Contact Luca Bertagna [lbertag@sandia.gov](mailto:lbertag@sandia.gov) diff --git a/.github/actions/test-all-scream/action.yml b/.github/actions/test-all-scream/action.yml new file mode 100644 index 000000000000..6ad60b8f31bd --- /dev/null +++ b/.github/actions/test-all-scream/action.yml @@ -0,0 +1,104 @@ +name: EAMxx standalone testing +description: | + Run EAMxx standalone testing with required configuration inputs. + More precisely, it launches test-all-scream with the proper flags. + See components/eamxx/scripts/test-all-scream for more details. + The configuration inputs are: + - build_type: the type of build to pass to test-all-scream. + - machine: the name of the machine to pass to test-all-scream + - generate: whether to generate baselines + - submit: whether to submit to cdash (unused if generate is 'true') + +inputs: + build_type: + description: 'Build type to run' + required: true + machine: + description: 'Machine name for test-all-scream' + required: true + generate: + description: 'Generate baselines instead of running tests' + required: true + default: 'false' + valid_values: + - 'true' + - 'false' + submit: + description: 'Submit results to cdash (unused if generate=true)' + required: true + default: 'false' + valid_values: + - 'true' + - 'false' + cmake-configs: + description: 'Semicolon-separated list of key=value pairs for CMake to pass to test-all-scream' + required: false + default: '' + +runs: + using: "composite" + steps: + - name: Set CA certificates env var + run: | + # Ensure the operating system is Linux + if [ "$(uname)" != "Linux" ]; then + echo "This action only supports Linux." + exit 1 + fi + # Set env var to be used in upload-artifacts phase + if [ -f /etc/debian_version ]; then + echo "NODE_EXTRA_CA_CERTS=/etc/ssl/certs/ca-certificates.crt" >> $GITHUB_ENV + elif [ -f /etc/redhat-release ] || [ -f /etc/centos-release ] || [ -f /etc/fedora-release ]; then + echo "NODE_EXTRA_CA_CERTS=/etc/pki/tls/certs/ca-bundle.crt" >> $GITHUB_ENV + else + echo "Unsupported Linux distribution" + exit 1 + fi + shell: sh + - name: Check repo presence + run: | + if [ ! -d ".git" ]; then + echo "Repository is not checked out. Please ensure the repository is checked out before running this action." + exit 1 + fi + shell: sh + - name: Print build specs + run: | + echo "Testing EAMxx standalone, for the following configuration:" + echo " build type : ${{ inputs.build_type }}" + echo " machine : ${{ inputs.machine }}" + echo " generate : ${{ inputs.generate }}" + echo " submit : ${{ inputs.submit }}" + echo " cmake-configs: ${{ inputs.cmake-configs }}" + shell: sh + - name: Run test-all-scream + working-directory: components/eamxx + run: | + cmd="./scripts/test-all-scream -m ${{ inputs.machine }} -t ${{inputs.build_type}} --baseline-dir AUTO -c EKAT_DISABLE_TPL_WARNINGS=ON" + if [ "${{ inputs.generate }}" = "true" ]; then + cmd+=" -g" + elif [ "${{ inputs.submit }}" = "true" ]; then + cmd+=" -s" + fi + + # If cmake-configs is non-empty, add tokens to test-all-scream via "-c key=val" + IFS=';' read -ra configs <<< "${{ inputs.cmake-configs }}" + for config in "${configs[@]}"; do + cmd+=" -c $config" + done + + # Print the full command, then run it + echo "test-all-scream call: $cmd" + $cmd + shell: sh + - name: Upload ctest logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: log-files-${{ inputs.build_type }}-${{ inputs.machine }} + path: | + components/eamxx/ctest-build/*/Testing/Temporary/Last*.log + components/eamxx/ctest-build/*/ctest_resource_file.json + components/eamxx/ctest-build/*/CMakeCache.txt + env: + NODE_EXTRA_CA_CERTS: ${{ env.NODE_EXTRA_CA_CERTS }} diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4fb88c2dc67b..94078c407205 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,9 +7,6 @@ updates: schedule: interval: "weekly" assignees: - - "rljacob" + - "bartgol" reviewers: - "mahf708" - - "bartgol" - labels: - - "AT: Integrate Without Testing" diff --git a/.github/workflows/e3sm-gh-ci-cime-tests.yml b/.github/workflows/e3sm-gh-ci-cime-tests.yml index 04f7fcb4ffcf..d973b90dc054 100644 --- a/.github/workflows/e3sm-gh-ci-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-cime-tests.yml @@ -2,7 +2,9 @@ name: gh on: pull_request: - branches: [ master ] + branches: + - master + - maint-3.0 paths: # first, yes to these - '.github/workflows/e3sm-gh-ci-cime-tests.yml' @@ -22,10 +24,14 @@ on: workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: ci: - if: false + if: ${{ github.repository == 'E3SM-Project/E3SM' }} runs-on: ubuntu-latest strategy: fail-fast: false @@ -36,7 +42,7 @@ jobs: - SMS_D_Ln5_P4.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.ghci-oci_gnu - ERS_Ld5_P4.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.ghci-oci_gnu.eamxx-prod container: - image: ghcr.io/e3sm-project/containers-ghci:ghci-0.1.0 + image: ghcr.io/e3sm-project/containers-ghci:ghci-0.2.1 steps: - diff --git a/.github/workflows/e3sm-gh-ci-w-cime-tests.yml b/.github/workflows/e3sm-gh-ci-w-cime-tests.yml index 48c367c8f625..0e30fc4e0a84 100644 --- a/.github/workflows/e3sm-gh-ci-w-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-w-cime-tests.yml @@ -1,8 +1,10 @@ -name: gh +name: gh-w on: pull_request: - branches: [ master ] + branches: + - master + - maint-3.0 paths-ignore: - 'mkdocs.yaml' - 'docs/**' @@ -11,10 +13,14 @@ on: workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: - ci-w: - if: ${{ github.event.repository.name == 'e3sm' }} + ci: + if: ${{ github.repository == 'E3SM-Project/E3SM' }} runs-on: ubuntu-latest strategy: fail-fast: false @@ -23,7 +29,7 @@ jobs: - SMS_D_Ld1_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu - ERS_Ld3_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu.allactive-wcprod_1850 container: - image: ghcr.io/e3sm-project/containers-ghci:ghci-0.1.0 + image: ghcr.io/e3sm-project/containers-ghci:ghci-0.2.1 steps: - diff --git a/.github/workflows/e3sm-gh-md-linter.yml b/.github/workflows/e3sm-gh-md-linter.yml index 424a871637b6..8be6f87893bc 100644 --- a/.github/workflows/e3sm-gh-md-linter.yml +++ b/.github/workflows/e3sm-gh-md-linter.yml @@ -7,11 +7,14 @@ on: branches: ["master"] paths: - '**/*.md' - # for now let's not lint files in eamxx - - '!components/eamxx/**/*.md' + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true jobs: linter: + if: ${{ github.repository == 'E3SM-Project/E3SM' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -22,7 +25,7 @@ jobs: with: files: '**/*.md' separator: "," - - uses: DavidAnson/markdownlint-cli2-action@v17 + - uses: DavidAnson/markdownlint-cli2-action@v19 if: steps.changed-files.outputs.any_changed == 'true' with: config: 'docs/.markdownlint.json' diff --git a/.github/workflows/e3sm-gh-pages.yml b/.github/workflows/e3sm-gh-pages.yml index ebd2ac9c1e97..543295a90302 100644 --- a/.github/workflows/e3sm-gh-pages.yml +++ b/.github/workflows/e3sm-gh-pages.yml @@ -4,9 +4,27 @@ on: # Runs every time master branch is updated push: branches: ["master"] + # But only if docs-related files are touched + paths: + - .github/workflows/e3sm-gh-pages.yml + - ./mkdocs.yml + - ./tools/*/mkdocs.yml + - ./tools/docs/** + - components/*/mkdocs.yaml + - components/*/docs/** + - components/eamxx/cime_config/namelist_defaults_scream.xml # Runs every time a PR is open against master pull_request: branches: ["master"] + # But only if docs-related files are touched + paths: + - .github/workflows/e3sm-gh-pages.yml + - ./mkdocs.yml + - ./tools/*/mkdocs.yml + - ./tools/docs/** + - components/*/mkdocs.yaml + - components/*/docs/** + - components/eamxx/cime_config/namelist_defaults_scream.xml workflow_dispatch: concurrency: @@ -15,7 +33,7 @@ concurrency: jobs: Build-and-Deploy-docs: - if: ${{ github.event.repository.name == 'e3sm' }} + if: ${{ github.repository == 'E3SM-Project/E3SM' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml b/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml deleted file mode 100644 index 8fe212886d9f..000000000000 --- a/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: mkatmsrffile - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - paths: - - 'components/eam/tools/mkatmsrffile/mkatmsrffile.py' - schedule: - - cron: '00 15 * * 2' - workflow_dispatch: - -jobs: - mkatmsrffile-test: - runs-on: ubuntu-latest - defaults: - run: - shell: bash -l {0} - outputs: - event_name: ${{ github.event_name }} - steps: - - - name: Repository checkout - uses: actions/checkout@v4 - with: - show-progress: false - submodules: false - - - name: Conda setup - uses: conda-incubator/setup-miniconda@v3 - with: - activate-environment: "envmkatmsrffile" - miniforge-variant: Mambaforge - miniforge-version: latest - use-mamba: true - mamba-version: "*" - channel-priority: strict - auto-update-conda: true - python-version: 3.11 - - - name: Install dependencies - run: | - echo $CONDA_PREFIX - mamba install -y nco xarray numba numpy netcdf4 - - - name: Run tests - working-directory: components/eam/tools/mkatmsrffile - run: | - echo $CONDA_PREFIX - wget https://web.lcrc.anl.gov/public/e3sm/inputdata/atm/cam/chem/trop_mozart/dvel/clim_soilw.nc - wget https://web.lcrc.anl.gov/public/e3sm/inputdata/atm/cam/chem/trop_mozart/dvel/regrid_vegetation.nc - wget https://web.lcrc.anl.gov/public/e3sm/inputdata/atm/cam/chem/trop_mozart/dvel/map_1x1_to_ne30pg2_traave_c20240903.nc - python mkatmsrffile.py --map_file=map_1x1_to_ne30pg2_traave_c20240903.nc --vegetation_file=regrid_vegetation.nc --soil_water_file=clim_soilw.nc --dst_grid=ne30pg2 - - mkatmsrffile-notify: - needs: mkatmsrffile-test - if: ${{ failure() && needs.mkatmsrffile-test.outputs.event_name != 'pull_request' }} - runs-on: ubuntu-latest - steps: - - name: Create issue - run: | - previous_issue_number=$(gh issue list \ - --label "$LABELS" \ - --json number \ - --jq '.[0].number') - if [[ -n $previous_issue_number ]]; then - gh issue comment "$previous_issue_number" \ - --body "$BODY" - else - gh issue create \ - --title "$TITLE" \ - --assignee "$ASSIGNEES" \ - --label "$LABELS" \ - --body "$BODY" - fi - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_REPO: ${{ github.repository }} - TITLE: mkatmsrffile test failure - ASSIGNEES: whannah1 - LABELS: bug,notify-mkatmsrffile-gh-action - BODY: | - Workflow failed! There's likely an issue in the mkatmsrffile tool! For more information, please see: - - Workflow URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} (number ${{ github.run_number }}, attempt ${{ github.run_attempt }}) - - Workflow SHA: ${{ github.sha }} diff --git a/.github/workflows/eamxx-gh-pages.yml b/.github/workflows/eamxx-gh-pages.yml deleted file mode 100644 index 2ce70d97843c..000000000000 --- a/.github/workflows/eamxx-gh-pages.yml +++ /dev/null @@ -1,89 +0,0 @@ -# This workflow aims to automatically rebuild eamxx documentation -# every time the master branch is updated on github and within every PR - -name: EAMxx Docs - -on: - # Runs every time master branch is updated - push: - branches: [ master ] - # Only if docs-related files are touched - paths: - - components/eamxx/mkdocs.yml - - components/eamxx/docs/** - - components/eamxx/cime_config/namelist_defaults_scream.xml - # Runs every time a PR is open against master - pull_request: - branches: [ master ] - # Only if docs-related files are touched - paths: - - components/eamxx/mkdocs.yml - - components/eamxx/docs/** - - components/eamxx/cime_config/namelist_defaults_scream.xml - - label: - types: - - created - - workflow_dispatch: - -concurrency: - # Prevent 2+ copies of this workflow from running concurrently - group: eamxx-docs-action - -jobs: - - eamxx-docs: - if: ${{ github.event.repository.name == 'scream' }} - runs-on: ubuntu-latest - - steps: - - name: Check out the repository - uses: actions/checkout@v4 - with: - persist-credentials: false - show-progress: false - # TODO: git rid of dependency on CIME - # TODO: another option to investigate is a sparse checkout. - # In the scream repo, all other components do not need to be checked out. - # And even in the upstream, we mainly need only components/xyz/docs (and a few more places). - submodules: true - - - name: Show action trigger - run: | - echo "= The job was automatically triggered by a ${{github.event_name}} event." - - - name: Set up Python 3.11 - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install Python deps - run: | - pip install mkdocs pymdown-extensions mkdocs-material mdutils mkdocs-bibtex - - - name: Generate EAMxx params docs - working-directory: components/eamxx/scripts - run: | - ./eamxx-params-docs-autogen - - - name: Build docs - working-directory: components/eamxx - run: | - mkdocs build --strict --verbose - - # only deploy to the main github page when there is a push to master - - if: ${{ github.event_name == 'push' }} - name: GitHub Pages action - uses: JamesIves/github-pages-deploy-action@v4 - with: - # Do not remove existing pr-preview pages - clean-exclude: pr-preview - folder: ./components/eamxx/site - - # If it's a PR from within the same repo, deploy to a preview page - - if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} - name: Preview docs - uses: rossjrw/pr-preview-action@v1 - with: - source-dir: components/eamxx/site/ diff --git a/.github/workflows/eamxx-sa-coverage.yml b/.github/workflows/eamxx-sa-coverage.yml new file mode 100644 index 000000000000..46e87f8b8b0c --- /dev/null +++ b/.github/workflows/eamxx-sa-coverage.yml @@ -0,0 +1,93 @@ +name: eamxx-sa-coverage + +on: + workflow_dispatch: + inputs: + submit: + description: 'Force cdash submission' + required: true + type: boolean + + # Add schedule trigger for nightly runs at midnight MT (Standard Time) + schedule: + - cron: '0 7 * * *' # Runs at 7 AM UTC, which is midnight MT during Standard Time + +concurrency: + # Two runs are in the same group if they are testing the same git ref + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + # Submit to cdash only for nightlies or if the user explicitly forced a submission via workflow dispatch + submit: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.submit) }} + +jobs: + gcc-openmp: + runs-on: [self-hosted, ghci-snl-cpu, gcc] + name: gcc-openmp / cov + steps: + - name: Check out the repository + uses: actions/checkout@v4 + with: + persist-credentials: false + show-progress: false + submodules: recursive + - name: Show action trigger + uses: ./.github/actions/show-workflow-trigger + - name: Run tests + uses: ./.github/actions/test-all-scream + with: + build_type: cov + machine: ghci-snl-cpu + generate: false + submit: ${{ env.submit }} + cmake-configs: Kokkos_ENABLE_OPENMP=ON + gcc-cuda: + runs-on: [self-hosted, ghci-snl-cuda, cuda, gcc] + name: gcc-cuda / cov + steps: + - name: Check out the repository + uses: actions/checkout@v4 + with: + persist-credentials: false + show-progress: false + submodules: recursive + - name: Show action trigger + uses: ./.github/actions/show-workflow-trigger + - name: Get CUDA Arch + run: | + # Ensure nvidia-smi is available + if ! command -v nvidia-smi &> /dev/null; then + echo "nvidia-smi could not be found. Please ensure you have Nvidia drivers installed." + exit 1 + fi + + # Get the GPU model from nvidia-smi, and set env for next step + gpu_model=$(nvidia-smi --query-gpu=name --format=csv,noheader | head -n 1) + case "$gpu_model" in + *"H100"*) + echo "Hopper=ON" >> $GITHUB_ENV + echo "CUDA_ARCH=90" >> $GITHUB_ENV + ARCH=90 + ;; + *"A100"*) + echo "Ampere=ON" >> $GITHUB_ENV + echo "CUDA_ARCH=80" >> $GITHUB_ENV + ;; + *"V100"*) + echo "Volta=ON" >> $GITHUB_ENV + echo "CUDA_ARCH=70" >> $GITHUB_ENV + ;; + *) + echo "Unsupported GPU model: $gpu_model" + exit 1 + ;; + esac + - name: Run tests + uses: ./.github/actions/test-all-scream + with: + build_type: cov + machine: ghci-snl-cuda + generate: false + submit: ${{ env.submit }} + cmake-configs: Kokkos_ARCH_HOPPER90=${{ env.Hopper }};Kokkos_ARCH_AMPERE80=${{ env.Ampere }};Kokkos_ARCH_VOLTA70=${{ env.Volta }};CMAKE_CUDA_ARCHITECTURES=${{ env.CUDA_ARCH }} diff --git a/.github/workflows/eamxx-sa-sanitizer.yml b/.github/workflows/eamxx-sa-sanitizer.yml new file mode 100644 index 000000000000..00f60f7da2e3 --- /dev/null +++ b/.github/workflows/eamxx-sa-sanitizer.yml @@ -0,0 +1,97 @@ +name: eamxx-sa-sanitizer + +on: + workflow_dispatch: + inputs: + submit: + description: 'Force cdash submission' + required: true + type: boolean + + # Add schedule trigger for nightly runs at midnight MT (Standard Time) + schedule: + - cron: '0 7 * * *' # Runs at 7 AM UTC, which is midnight MT during Standard Time + +concurrency: + # Two runs are in the same group if they are testing the same git ref + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + # Submit to cdash only for nightlies or if the user explicitly forced a submission via workflow dispatch + submit: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.submit) }} + +jobs: + gcc-openmp: + runs-on: [self-hosted, ghci-snl-cpu, gcc] + name: gcc-openmp / valg + steps: + - name: Check out the repository + uses: actions/checkout@v4 + with: + persist-credentials: false + show-progress: false + submodules: recursive + - name: Show action trigger + uses: ./.github/actions/show-workflow-trigger + - name: Run tests + uses: ./.github/actions/test-all-scream + with: + build_type: valg + machine: ghci-snl-cpu + generate: false + submit: ${{ env.submit }} + cmake-configs: Kokkos_ENABLE_OPENMP=ON + gcc-cuda: + runs-on: [self-hosted, ghci-snl-cuda, cuda, gcc] + strategy: + fail-fast: false + matrix: + build_type: [csm, csr, csi, css] + name: gcc-cuda / ${{ matrix.build_type }} + steps: + - name: Check out the repository + uses: actions/checkout@v4 + with: + persist-credentials: false + show-progress: false + submodules: recursive + - name: Show action trigger + uses: ./.github/actions/show-workflow-trigger + - name: Get CUDA Arch + run: | + # Ensure nvidia-smi is available + if ! command -v nvidia-smi &> /dev/null; then + echo "nvidia-smi could not be found. Please ensure you have Nvidia drivers installed." + exit 1 + fi + + # Get the GPU model from nvidia-smi, and set env for next step + gpu_model=$(nvidia-smi --query-gpu=name --format=csv,noheader | head -n 1) + case "$gpu_model" in + *"H100"*) + echo "Hopper=ON" >> $GITHUB_ENV + echo "CUDA_ARCH=90" >> $GITHUB_ENV + ARCH=90 + ;; + *"A100"*) + echo "Ampere=ON" >> $GITHUB_ENV + echo "CUDA_ARCH=80" >> $GITHUB_ENV + ;; + *"V100"*) + echo "Volta=ON" >> $GITHUB_ENV + echo "CUDA_ARCH=70" >> $GITHUB_ENV + ;; + *) + echo "Unsupported GPU model: $gpu_model" + exit 1 + ;; + esac + - name: Run tests + uses: ./.github/actions/test-all-scream + with: + build_type: ${{ matrix.build_type }} + machine: ghci-snl-cuda + generate: false + submit: ${{ env.submit }} + cmake-configs: Kokkos_ARCH_HOPPER90=${{ env.Hopper }};Kokkos_ARCH_AMPERE80=${{ env.Ampere }};Kokkos_ARCH_VOLTA70=${{ env.Volta }};CMAKE_CUDA_ARCHITECTURES=${{ env.CUDA_ARCH }} diff --git a/.github/workflows/eamxx-sa-testing.yml b/.github/workflows/eamxx-sa-testing.yml new file mode 100644 index 000000000000..7bc20aeaca5b --- /dev/null +++ b/.github/workflows/eamxx-sa-testing.yml @@ -0,0 +1,151 @@ +name: eamxx-sa + +on: + # Runs on PRs against master + pull_request: + branches: [ master ] + types: [opened, synchronize, ready_for_review, reopened] + paths: + # first, yes to these + - '.github/workflows/eamxx-sa-testing.yml' + - 'cime_config/machine/config_machines.xml' + - 'components/eamxx/**' + - 'components/homme/**' + - 'externals/ekat' + - 'externals/mam4xx' + - 'externals/haero' + - 'externals/scorpio' + # second, no to these + - '!components/eamxx/docs/**' + - '!components/eamxx/mkdocs.yml' + + # Manual run is used to bless + workflow_dispatch: + inputs: + job_to_run: + description: 'Job to run' + required: true + type: choice + options: + - gcc-openmp + - gcc-cuda + - all + bless: + description: 'Generate baselines' + required: true + type: boolean + submit: + description: 'Force cdash submission' + required: true + type: boolean + + # Add schedule trigger for nightly runs at midnight MT (Standard Time) + schedule: + - cron: '0 7 * * *' # Runs at 7 AM UTC, which is midnight MT during Standard Time + +concurrency: + # Two runs are in the same group if they are testing the same git ref + # - if trigger=pull_request, the ref is refs/pull//merge + # - for other triggers, the ref is the branch tested + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + # Submit to cdash only for nightlies or if the user explicitly forced a submission via workflow dispatch + submit: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.submit) }} + generate: ${{ github.event_name == 'workflow_dispatch' && inputs.bless }} + +jobs: + gcc-openmp: + if: | + ${{ + github.event_name != 'workflow_dispatch' || + ( + github.event.inputs.job_to_run == 'gcc-openmp' || + github.event.inputs.job_to_run == 'all' + ) + }} + runs-on: [self-hosted, ghci-snl-cpu, gcc] + strategy: + fail-fast: false + matrix: + build_type: [sp, dbg, fpe, opt] + name: gcc-openmp / ${{ matrix.build_type }} + steps: + - name: Check out the repository + uses: actions/checkout@v4 + with: + persist-credentials: false + show-progress: false + submodules: recursive + - name: Show action trigger + uses: ./.github/actions/show-workflow-trigger + - name: Run tests + uses: ./.github/actions/test-all-scream + with: + build_type: ${{ matrix.build_type }} + machine: ghci-snl-cpu + generate: ${{ env.generate }} + submit: ${{ env.submit }} + cmake-configs: Kokkos_ENABLE_OPENMP=ON + gcc-cuda: + if: | + ${{ + github.event_name != 'workflow_dispatch' || + ( + github.event.inputs.job_to_run == 'gcc-cuda' || + github.event.inputs.job_to_run == 'all' + ) + }} + runs-on: [self-hosted, ghci-snl-cuda, cuda, gcc] + strategy: + fail-fast: false + matrix: + build_type: [sp, dbg, opt] + name: gcc-cuda / ${{ matrix.build_type }} + steps: + - name: Check out the repository + uses: actions/checkout@v4 + with: + persist-credentials: false + show-progress: false + submodules: recursive + - name: Show action trigger + uses: ./.github/actions/show-workflow-trigger + - name: Get CUDA Arch + run: | + # Ensure nvidia-smi is available + if ! command -v nvidia-smi &> /dev/null; then + echo "nvidia-smi could not be found. Please ensure you have Nvidia drivers installed." + exit 1 + fi + + # Get the GPU model from nvidia-smi, and set env for next step + gpu_model=$(nvidia-smi --query-gpu=name --format=csv,noheader | head -n 1) + case "$gpu_model" in + *"H100"*) + echo "Hopper=ON" >> $GITHUB_ENV + echo "CUDA_ARCH=90" >> $GITHUB_ENV + ARCH=90 + ;; + *"A100"*) + echo "Ampere=ON" >> $GITHUB_ENV + echo "CUDA_ARCH=80" >> $GITHUB_ENV + ;; + *"V100"*) + echo "Volta=ON" >> $GITHUB_ENV + echo "CUDA_ARCH=70" >> $GITHUB_ENV + ;; + *) + echo "Unsupported GPU model: $gpu_model" + exit 1 + ;; + esac + - name: Run tests + uses: ./.github/actions/test-all-scream + with: + build_type: ${{ matrix.build_type }} + machine: ghci-snl-cuda + generate: ${{ env.generate }} + submit: ${{ env.submit }} + cmake-configs: Kokkos_ARCH_HOPPER90=${{ env.Hopper }};Kokkos_ARCH_AMPERE80=${{ env.Ampere }};Kokkos_ARCH_VOLTA70=${{ env.Volta }};CMAKE_CUDA_ARCHITECTURES=${{ env.CUDA_ARCH }} diff --git a/.github/workflows/eamxx-scripts-tests.yml b/.github/workflows/eamxx-scripts-tests.yml new file mode 100644 index 000000000000..2cdd6f8758f8 --- /dev/null +++ b/.github/workflows/eamxx-scripts-tests.yml @@ -0,0 +1,56 @@ +name: eamxx-scripts + +on: + # Runs on PRs against master, but only if they touch eamxx scripts + pull_request: + branches: [ master ] + types: [opened, synchronize, ready_for_review, reopened] + paths: + - 'components/eamxx/scripts/**' + - 'components/eamxx/cime_config/**' + - '.github/workflows/eamxx-scripts-tests.yml' + + # Manual run for debug purposes only + workflow_dispatch: + inputs: + submit: + description: 'Force cdash submission' + required: true + type: boolean + + # Add schedule trigger for nightly runs at midnight MT (Standard Time) + schedule: + - cron: '0 7 * * *' # Runs at 7 AM UTC, which is midnight MT during Standard Time + +concurrency: + # Two runs are in the same group if they are testing the same git ref + # - if trigger=pull_request, the ref is refs/pull//merge + # - for other triggers, the ref is the branch tested + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + # Submit to cdash only for nightlies or if the user explicitly forced a submission via workflow dispatch + submit: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.submit) }} + +jobs: + cpu-gcc: + runs-on: [self-hosted, gcc, ghci-snl-cpu] + steps: + - name: Check out the repository + uses: actions/checkout@v4 + with: + persist-credentials: false + show-progress: false + submodules: recursive + - name: Show action trigger + uses: ./.github/actions/show-workflow-trigger + - name: Run test + run: | + cd components/eamxx + if [ "${{ env.submit }}" == "true" ]; then + ./scripts/scripts-ctest-driver -s -m ghci-snl-cpu + else + ./scripts/scripts-tests -f -m ghci-snl-cpu + ./scripts/cime-nml-tests + fi diff --git a/.github/workflows/eamxx-v1-testing.yml b/.github/workflows/eamxx-v1-testing.yml new file mode 100644 index 000000000000..e738a4adcb90 --- /dev/null +++ b/.github/workflows/eamxx-v1-testing.yml @@ -0,0 +1,126 @@ +name: eamxx-v1 + +on: + # Runs on PRs against master + pull_request: + branches: [ master ] + types: [opened, synchronize, ready_for_review, reopened] + paths: + # first, yes to these + - '.github/workflows/eamxx-v1-testing.yml' + - 'cime_config/machine/config_machines.xml' + - 'components/eamxx/**' + - 'components/homme/**' + - 'externals/ekat' + - 'externals/mam4xx' + - 'externals/haero' + - 'externals/scorpio' + # second, no to these + - '!components/eamxx/docs/**' + - '!components/eamxx/mkdocs.yml' + + # Manual run is used to bless + workflow_dispatch: + inputs: + job_to_run: + description: 'Job to run' + required: true + type: choice + options: + - cpu-gcc + - all + bless: + description: 'Generate baselines' + required: true + type: boolean + +concurrency: + # Two runs are in the same group if they are testing the same git ref + # - if trigger=pull_request, the ref is refs/pull//merge + # - for other triggers, the ref is the branch tested + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + # Submit to cdash only for nightlies or if the user explicitly forced a submission via workflow dispatch + submit: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.submit) }} + # Generate only if user requested via workflow_dispatch + generate: ${{ github.event_name == 'workflow_dispatch' && inputs.bless }} + # Correct case folder suffix for generate/compare, used to find files to upload as artifacts + folder_suffix: ${{ github.event_name == 'workflow_dispatch' && inputs.bless && '.G' || '.C' }} + # Compare/generate flags for create_test + flags: ${{ github.event_name == 'workflow_dispatch' && inputs.bless && '-o -g -b master' || '-c -b master' }} + +jobs: + cpu-gcc: + if: | + ${{ + github.event_name != 'workflow_dispatch' || + ( + github.event.inputs.job_to_run == 'cpu-gcc' || + github.event.inputs.job_to_run == 'all' + ) + }} + runs-on: [self-hosted, gcc, ghci-snl-cpu] + strategy: + matrix: + test: + - full_name: ERS_Ln9.ne4_ne4.F2000-SCREAMv1-AQP1.ghci-snl-cpu_gnu.eamxx-output-preset-2 + short_name: ERS_Ln9.ne4_ne4.F2000-SCREAMv1-AQP1.eamxx-output-preset-2 + - full_name: ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.ghci-snl-cpu_gnu.eamxx-dpxx-arm97 + short_name: ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.eamxx-dpxx-arm97 + - full_name: ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.ghci-snl-cpu_gnu.eamxx-small_kernels--eamxx-output-preset-5 + short_name: ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.eamxx-small_kernels--eamxx-output-preset-5 + - full_name: SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.ghci-snl-cpu_gnu.eamxx-mam4xx-all_mam4xx_procs + short_name: SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.eamxx-mam4xx-all_mam4xx_procs + - full_name: SMS_Ln3_P4.ne4pg2_oQU480.F2010-MMF2.ghci-snl-cpu_gnu + short_name: SMS_Ln3_P4.ne4pg2_oQU480.F2010-MMF2 + fail-fast: false + name: cpu-gcc / ${{ matrix.test.short_name }} + steps: + - name: Check out the repository + uses: actions/checkout@v4 + with: + persist-credentials: false + show-progress: false + submodules: recursive + - name: Show action trigger + uses: ./.github/actions/show-workflow-trigger + - name: Set CA certificates env var + run: | + # Ensure the operating system is Linux + if [ "$(uname)" != "Linux" ]; then + echo "This action only supports Linux." + exit 1 + fi + # Set env var to be used in upload-artifacts phase + if [ -f /etc/debian_version ]; then + echo "NODE_EXTRA_CA_CERTS=/etc/ssl/certs/ca-certificates.crt" >> $GITHUB_ENV + elif [ -f /etc/redhat-release ] || [ -f /etc/centos-release ] || [ -f /etc/fedora-release ]; then + echo "NODE_EXTRA_CA_CERTS=/etc/pki/tls/certs/ca-bundle.crt" >> $GITHUB_ENV + else + echo "Unsupported Linux distribution" + exit 1 + fi + - name: Run test + run: | + ./cime/scripts/create_test ${{ matrix.test.full_name }} ${{ env.flags }} --wait + - name: Upload case files + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.test.full_name }} + path: | + /projects/e3sm/scratch/${{ matrix.test.full_name }}${{ env.folder_suffix }}*/TestStatus.log + /projects/e3sm/scratch/${{ matrix.test.full_name }}${{ env.folder_suffix }}*/bld/*.bldlog.* + /projects/e3sm/scratch/${{ matrix.test.full_name }}${{ env.folder_suffix }}*/bld/case2bld/*.bldlog.* + /projects/e3sm/scratch/${{ matrix.test.full_name }}${{ env.folder_suffix }}*/run/*.log.* + /projects/e3sm/scratch/${{ matrix.test.full_name }}${{ env.folder_suffix }}*/run/case2run/*.log.* + /projects/e3sm/scratch/${{ matrix.test.full_name }}${{ env.folder_suffix }}*/run/*.cprnc.out + /projects/e3sm/scratch/${{ matrix.test.full_name }}${{ env.folder_suffix }}*/run/case2run/*.cprnc.out + env: + NODE_EXTRA_CA_CERTS: ${{ env.NODE_EXTRA_CA_CERTS }} + - name: Clean up + if: ${{ always() }} + run: | + rm -rf /projects/e3sm/scratch/${{ matrix.test.full_name }}* diff --git a/.github/workflows/eamxx_default_files.yml b/.github/workflows/eamxx_default_files.yml deleted file mode 100644 index d39717589914..000000000000 --- a/.github/workflows/eamxx_default_files.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: inputdata - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - paths: - - 'components/eamxx/cime_config/namelist_defaults_scream.xml' - schedule: - - cron: '00 00 * * *' - workflow_dispatch: - -jobs: - scream-defaults: - if: false - runs-on: ubuntu-latest - outputs: - event_name: ${{ github.event_name }} - steps: - - name: Check out the repository - uses: actions/checkout@v4 - with: - show-progress: false - submodules: false - - name: Set up Python 3.11 - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - name: Run unit tests - working-directory: components/eamxx/cime_config/ - run: | - python -m unittest tests/eamxx_default_files.py -v - - notify-scream-defaults: - needs: scream-defaults - if: ${{ failure() && needs.scream-defaults.outputs.event_name != 'pull_request' }} - runs-on: ubuntu-latest - steps: - - name: Create issue - run: | - previous_issue_number=$(gh issue list \ - --label "$LABELS" \ - --json number \ - --jq '.[0].number') - if [[ -n $previous_issue_number ]]; then - gh issue comment "$previous_issue_number" \ - --body "$BODY" - else - gh issue create \ - --title "$TITLE" \ - --assignee "$ASSIGNEES" \ - --label "$LABELS" \ - --body "$BODY" - fi - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_REPO: ${{ github.repository }} - TITLE: Inputdata server file missing - ASSIGNEES: mahf708,bartgol - LABELS: bug,input file,notify-file-gh-action - BODY: | - Workflow failed! There's likely a missing file specified in the configs! For more information, please see: - - Workflow URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} (number ${{ github.run_number }}, attempt ${{ github.run_attempt }}) - - Workflow SHA: ${{ github.sha }} diff --git a/CITATION.cff b/CITATION.cff index 9542a7d01ef4..78099055fc35 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -9,7 +9,7 @@ authors: - given-names: E3SM family-names: Project version: 3.0.0 -doi: 10.11578/E3SM/dc.20240301.3 +doi: 10.11578/E3SM/dc.20240930.1 repository-code: 'https://github.com/E3SM-Project/E3SM' url: 'https://e3sm.org' license: BSD-3-Clause diff --git a/LICENSE b/LICENSE index d74a2aa127a0..247287ab4c00 100644 --- a/LICENSE +++ b/LICENSE @@ -2,7 +2,7 @@ Except for the separable pieces descibed below, E3SM is released under the following 3-Clause BSD Open Source license. ******************************************************************************* -Copyright ©2023, UChicago Argonne, LLC All Rights Reserved +Copyright 2024, UChicago Argonne, LLC All Rights Reserved Software Name: Energy Exascale Earth System Model (E3SM) @@ -55,15 +55,29 @@ GPTL share/timing author non-commeric MCT externals/mct ANL BSD YAKL externals/YAKL author BSD cub externals/cub author, NVIDIA BSD -kokkos externals/kokkos SNL BSD +kokkos externals/ekat SNL BSD +haero externals/haero SNL/Battelle BSD +mam4xx externals/mam4xx SNL/Battelle BSD Ocean/Ice under components/ ----------- ----------------- CICE cice LANL BSD MPAS Framework mpas-framework LANL BSD -MPAS Ocean mpas-ocean LANL BSD -MPAS SeaIce mpas-seaice LANL BSD -MPAS-Albany LandIce mpas-albany-landice LANL, SNL BSD +FFTW mpas-ocean/src/FFTW author/MIT GPL +MARBL mpas-ocean/src/MARBL NCAR BSD +SHTNS mpas-ocean/src/SHTNS CeCILL GPL +cvmix mpas-ocean/src/cvmix NCAR LGPL +gotm mpas-ocean/src/gotm authors GPL +ppr mpas-ocean/src/ppr author custom +Icepack mpas-seaice/src/icepack LANL BSD + +Waves under components/ww3 +----------- -------------------- +WW3 src/WW3 NWS/NOAA custom + +Land-ice under components/mpas-albany-landice +----------- ------------------------------------ +SeaLevelModel src/SeaLevelModel author MIT Land under components/elm/src ----------- ------------------------ @@ -91,7 +105,7 @@ HOMMEXX components/homme/src/share/cxx SNL BSD Actual copyright holder for above Institutions: NCAR = University Corporation for Atmospheric Research -LANL = Los Alamos National Security, LLC +LANL = Los Alamos National Security, LLC, Triad National Security, LLC SNL = National Technology & Engineering Solutions of Sandia, LLC LBNL = The Regents of the University of California, through Lawrence Berkeley National Laboratory diff --git a/README.md b/README.md index 192c288d78e1..13c927b8c7f1 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ + + [![E3SM Logo](https://e3sm.org/wp-content/themes/e3sm/assets/images/e3sm-logo.png)](https://e3sm.org) Energy Exascale Earth System Model (E3SM) @@ -9,14 +11,14 @@ the most challenging and demanding climate-change research problems and Department of Energy mission needs while efficiently using DOE Leadership Computing Facilities. -DOI: [10.11578/E3SM/dc.20240301.3](http://dx.doi.org/10.11578/E3SM/dc.20240301.3) +DOI: [10.11578/E3SM/dc.20240930.1](http://dx.doi.org/10.11578/E3SM/dc.20240930.1) Please visit the [project website](https://e3sm.org) or our [Confluence site](https://acme-climate.atlassian.net/wiki/spaces/DOC/overview) for further details. For questions about the model, use [Github Discussions](https://github.com/E3SM-Project/E3SM/discussions). -See our Github-hosted documentation at [https://e3sm-project.github.io/E3SM/](https://e3sm-project.github.io/E3SM/). +See our Github-hosted documentation at [https://docs.e3sm.org/E3SM](https://docs.e3sm.org/E3SM/). Table of Contents -------------------------------------------------------------------------------- diff --git a/cime_config/allactive/config_compsets.xml b/cime_config/allactive/config_compsets.xml index e66f1e473d85..d5a52a8bac38 100755 --- a/cime_config/allactive/config_compsets.xml +++ b/cime_config/allactive/config_compsets.xml @@ -383,42 +383,52 @@ CRYO1850 - 1850SOI_EAM%CMIP6_ELM%SPBC_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV + 1850SOI_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV CRYO1850-4xCO2 - 1850SOI_EAM%CMIP6-4xCO2_ELM%SPBC_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV + 1850SOI_EAM%CMIP6-4xCO2_ELM%CNPRDCTCBCTOP_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV + + + + CRYO1850-1pctCO2 + 1850SOI_EAM%CMIP6-1pctCO2_ELM%CNPRDCTCBCTOP_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV CRYO1950 - 1950SOI_EAM%CMIP6_ELM%SPBC_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV + 1950SOI_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV CRYO1850-DISMF - 1850SOI_EAM%CMIP6_ELM%SPBC_MPASSI%DIB_MPASO%IBDISMF_MOSART_SGLC_SWAV + 1850SOI_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI%DIB_MPASO%IBDISMF_MOSART_SGLC_SWAV CRYO1950-DISMF - 1950SOI_EAM%CMIP6_ELM%SPBC_MPASSI%DIB_MPASO%IBDISMF_MOSART_SGLC_SWAV + 1950SOI_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI%DIB_MPASO%IBDISMF_MOSART_SGLC_SWAV CRYO20TR - 20TRSOI_EAM%CMIP6_ELM%SPBC_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV + 20TRSOI_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV + + + + CRYOSSP245 + SSP245SOI_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV CRYOSSP585 - SSP585SOI_EAM%CMIP6_ELM%SPBC_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV + SSP585SOI_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV CRYOSSP370 - SSP370SOI_EAM%CMIP6_ELM%SPBC_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV + SSP370SOI_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV diff --git a/cime_config/allactive/config_pesall.xml b/cime_config/allactive/config_pesall.xml index 3392ad71ca10..a12a89d5220e 100644 --- a/cime_config/allactive/config_pesall.xml +++ b/cime_config/allactive/config_pesall.xml @@ -843,6 +843,29 @@ + + + + -compset WCYCL*/CRYO* -res SOwISC12to30E3r3* on 52 nodes pure-MPI, ~8.5 sypd + + 1408 + 384 + 384 + 1024 + 1920 + 1408 + + + 0 + 1024 + 1024 + 0 + 1408 + 0 + + + + @@ -1146,14 +1169,14 @@ improv: any compset on ne30np4 grid - -4 - -4 - -4 - -4 - -4 - -4 - -4 - -4 + -6 + -6 + -6 + -6 + -6 + -6 + -6 + -6 @@ -1774,6 +1797,21 @@ + + + allactive+pm-cpu: default, 1 node, 96 tasks, 1 thread + + 96 + 96 + 96 + 96 + 96 + 96 + 96 + 96 + + + @@ -2027,6 +2065,21 @@ + + + improv+allactive: RRM-WCYCL on 6 nodes + + -6 + -6 + -6 + -6 + -6 + -6 + -6 + -6 + + + cmod016b64x1 s=2.4 @@ -2164,6 +2217,149 @@ + + + + allactive+chrysalis: v3.NARRM tri-grid on 10 nodes ~1 sypd + + 576 + 576 + 576 + 576 + 576 + 64 + + + 576 + + + + allactive+chrysalis: v3.NARRM tri-grid on 20 nodes ~2 sypd + + 1152 + 1152 + 768 + 768 + 384 + 128 + + + 1152 + 384 + 384 + + + + allactive+chrysalis: v3.NARRM tri-grid on 30 nodes ~3 sypd + + 1792 + 1792 + 1280 + 1280 + 512 + 128 + + + 1792 + 512 + 512 + + + + allactive+chrysalis: v3.NARRM tri-grid on 40 nodes ~4 sypd + + 2368 + 2368 + 1408 + 1408 + 960 + 192 + + + 2368 + 1408 + + + + allactive+chrysalis: v3.NARRM tri-grid on 50 nodes ~5 sypd + + 3008 + 3008 + 1856 + 1856 + 1152 + 192 + + + 3008 + 1152 + 1152 + + + + allactive+chrysalis: v3.NARRM tri-grid on 64 nodes ~6 sypd + + 3840 + 3840 + 2304 + 2304 + 1536 + 256 + + + 3840 + 1536 + 1536 + + + + + + allactive+anvil: v3.NARRM tri-grid on 64 nodes ~1.8 sypd + + 2160 + 2160 + 2160 + 2160 + 2160 + 144 + + + 2160 + + + + allactive+anvil: v3.NARRM tri-grid on 96 nodes ~2.5 sypd + + 3240 + 3240 + 1080 + 1080 + 2160 + 216 + + + 3240 + 2160 + 2160 + + + + allactive+anvil: v3.NARRM tri-grid on 128 nodes ~3.2 sypd + + 4320 + 4320 + 4320 + 4320 + 4320 + 288 + + + 4320 + + + + diff --git a/cime_config/config_grids.xml b/cime_config/config_grids.xml index 98ee3313cc6d..45f886588e2b 100755 --- a/cime_config/config_grids.xml +++ b/cime_config/config_grids.xml @@ -416,6 +416,16 @@ RRSwISC6to18E3r5 + + T62 + T62 + SOwISC12to30E3r3 + rx1 + null + null + SOwISC12to30E3r3 + + TL319 TL319 @@ -656,6 +666,16 @@ RRSwISC6to18E3r5 + + TL319 + TL319 + SOwISC12to30E3r3 + JRA025 + null + null + SOwISC12to30E3r3 + + TL319 TL319 @@ -1120,6 +1140,16 @@ IcoswISC30E3r5 + + ne0np4_CAx32v1.pg2 + ne0np4_CAx32v1.pg2 + ne0np4_CAx32v1.pg2 + r0125 + null + null + oRRS18to6v3 + + ne0np4_northamericax4v1 r0125 @@ -1390,6 +1420,16 @@ RRSwISC6to18E3r5 + + ne30np4.pg2 + ne30np4.pg2 + SOwISC12to30E3r3 + r05 + null + null + SOwISC12to30E3r3 + + ne0np4_northamericax4v1 r0125 @@ -1919,6 +1959,16 @@ IcoswISC30E3r5 + + TL319 + TL319 + oQU240wLI + JRA025 + mpas.gis20km + null + oQU240wLI + + TL319 TL319 @@ -1929,6 +1979,16 @@ IcoswISC30E3r5 + + TL319 + TL319 + IcoswISC30E3r5 + JRA025 + mpas.gis1to10kmR2 + null + IcoswISC30E3r5 + + ne30np4.pg2 r05 @@ -1999,6 +2059,26 @@ IcoswISC30E3r5 + + ERA5r025 + r05 + IcoswISC30E3r5 + r05 + mpas.gis1to10kmR2 + null + IcoswISC30E3r5 + + + + ERA5r025 + r025 + IcoswISC30E3r5 + r025 + mpas.gis1to10kmR2 + null + IcoswISC30E3r5 + + ne120np4.pg2 r0125 @@ -2355,6 +2435,16 @@ EC30to60E2r2 + + ne30np4.pg2 + r05 + IcoswISC30E3r5 + r05 + null + wQU225Icos30E3r5 + IcoswISC30E3r5 + + ne30np4.pg2 ne30np4.pg2 @@ -2447,6 +2537,16 @@ RRSwISC6to18E3r5 + + ne30np4.pg2 + r05 + SOwISC12to30E3r3 + r05 + null + null + SOwISC12to30E3r3 + + ne30np4.pg2 r05 @@ -2758,6 +2858,7 @@ $DIN_LOC_ROOT/share/domains/domain.lnd.T62_ECwISC30to60E2r1.201007.nc $DIN_LOC_ROOT/share/domains/domain.lnd.T62_IcoswISC30E3r5.231121.nc $DIN_LOC_ROOT/share/domains/domain.lnd.T62_RRSwISC6to18E3r5.240328.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.T62_SOwISC12to30E3r3.240808.nc T62 is Gaussian grid: @@ -2816,11 +2917,21 @@ $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_IcosXISC30E3r7.240326.nc $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_RRSwISC6to18E3r5.240328.nc $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_RRSwISC6to18E3r5.240328.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_SOwISC12to30E3r3.240808.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_SOwISC12to30E3r3.240808.nc $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_oRRS18to6v3.220124.nc $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_oRRS18to6v3.220124.nc TL319 is JRA lat/lon grid: + + 1440 + 721 + $DIN_LOC_ROOT/share/domains/domain.lnd.ERA5r025_IcoswISC30E3r5.240903.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.ERA5r025_IcoswISC30E3r5.240903.nc + ERA5r025 is the lat/lon cap grid used by ERA5 data: + + @@ -2929,6 +3040,8 @@ $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_IcosXISC30E3r7.240326.nc $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_RRSwISC6to18E3r5.240328.nc $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_RRSwISC6to18E3r5.240328.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_SOwISC12to30E3r3.240808.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_SOwISC12to30E3r3.240808.nc $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_gx1v6.190806.nc $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_gx1v6.190806.nc ne30np4.pg2 is Spectral Elem 1-deg grid w/ 2x2 FV physics grid per element: @@ -3249,6 +3362,13 @@ RRSwISC6to18E3r5 is a MPAS ocean grid generated with the jigsaw/compass process using a mesh density function that is roughly proportional to the Rossby radius of deformation, with 18 km gridcells at low and 6 km gridcells at high latitudes. Additionally, it has ocean in ice-shelf cavities: + + 807630 + 1 + $DIN_LOC_ROOT/share/domains/domain.ocn.SOwISC12to30E3r3.240808.nc + SOwISC12to30E3r3 is a MPAS ocean grid generated with the jigsaw/compass process using XXXXX. Additionally, it has ocean in ice-shelf cavities: + + @@ -3285,6 +3405,8 @@ $DIN_LOC_ROOT/share/domains/domain.lnd.r05_IcosXISC30E3r7.240326.nc $DIN_LOC_ROOT/share/domains/domain.lnd.r05_RRSwISC6to18E3r5.240328.nc $DIN_LOC_ROOT/share/domains/domain.lnd.r05_RRSwISC6to18E3r5.240328.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.r05_SOwISC12to30E3r3.240808.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.r05_SOwISC12to30E3r3.240808.nc $DIN_LOC_ROOT/share/domains/domain.lnd.r05_gx1v6.191014.nc r05 is 1/2 degree river routing grid: @@ -3394,6 +3516,13 @@ $DIN_LOC_ROOT/share/domains/domain.ocn.wQU225EC30to60E2r2.220224.nc WW3 unstructured QU 225km global grid with EC30to60E2r2 coastlines + + + 97988 + 1 + $DIN_LOC_ROOT/share/domains/domain.ocn.wQU225Icos30E3r5.240910.nc + WW3 unstructured QU 225km global grid with ICOS30 coastlines + @@ -3467,6 +3596,14 @@ 1-deg with 1/4-deg over North America (version 1) pg2: + + 67872 + 1 + $DIN_LOC_ROOT/share/domains/domain.lnd.CAne32x32v1pg2_oRRS18to6v3.220505.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.CAne32x32v1pg2_oRRS18to6v3.220505.nc + 1-deg with 3 km over California (version 1; as described in Zhang et al. (2024)) pg2: + + 97200 1 @@ -3555,6 +3692,7 @@ ATM2ROF_SMAPNAME ATM2WAV_SMAPNAME OCN2WAV_SMAPNAME + WAV2OCN_SMAPNAME ICE2WAV_SMAPNAME ROF2OCN_LIQ_RMAPNAME @@ -3799,6 +3937,16 @@ cpl/gridmaps/ne30pg2/map_ne30pg2_to_RRSwISC6to18E3r5_trfvnp2.20240328.nc + + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_traave.20240808.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trbilin.20240808.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3-nomask_trbilin.20240808.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_ne30pg2_traave.20240808.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_ne30pg2_traave.20240808.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trfvnp2.20240808.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trfvnp2.20240808.nc + + cpl/gridmaps/ne30pg3/map_ne30pg3_to_oEC60to30v3_mono.200331.nc cpl/gridmaps/ne30pg3/map_ne30pg3_to_oEC60to30v3_bilin.200331.nc @@ -4417,6 +4565,13 @@ cpl/gridmaps/antarcticax4v1np4/map_antarcticax4v1pg2_to_r0125_mono.20200925.nc + + cpl/gridmaps/CAx32v1pg2/map_CAne32x32v1pg2_to_r0125_traave.20220430.nc + cpl/gridmaps/CAx32v1pg2/map_CAne32x32v1pg2_to_r0125_traave.20220430.nc + cpl/gridmaps/CAx32v1pg2/map_CAne32x32v1pg2_to_r0125_traave.20220430.nc + cpl/gridmaps/CAx32v1pg2/map_r0125_to_CAne32x32v1pg2_traave.20220430.nc + + cpl/gridmaps/T62/map_T62_TO_gx3v7_aave.130322.nc cpl/gridmaps/T62/map_T62_TO_gx3v7_blin.130322.nc @@ -4601,6 +4756,14 @@ cpl/gridmaps/RRSwISC6to18E3r5/map_RRSwISC6to18E3r5_to_T62_traave.20240328.nc + + cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3_traave.20240808.nc + cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3-nomask_trbilin.20240808.nc + cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3_esmfpatch.20240808.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_T62_traave.20240808.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_T62_traave.20240808.nc + + cpl/gridmaps/TL319/map_TL319_to_oQU240wLI_traave.20240509.nc cpl/gridmaps/TL319/map_TL319_to_oQU240wLI-nomask_trbilin.20240509.nc @@ -4753,6 +4916,14 @@ cpl/gridmaps/RRSwISC6to18E3r5/map_RRSwISC6to18E3r5_to_TL319_traave.20240328.nc + + cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3_traave.20240808.nc + cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3-nomask_trbilin.20240808.nc + cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3_esmfpatch.20240808.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_TL319_traave.20240808.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_TL319_traave.20240808.nc + + cpl/gridmaps/TL319/map_TL319_to_oRRS18to6v3_aave.220124.nc cpl/gridmaps/TL319/map_TL319_to_oRRS18to6v3_bilin.220124.nc @@ -4768,6 +4939,38 @@ cpl/gridmaps/TL319/map_r05_to_TL319_traave.20240212.nc + + cpl/gridmaps/ERA5r025/map_ERA5r025_to_IcoswISC30E3r5_traave.20240903.nc + cpl/gridmaps/ERA5r025/map_ERA5r025_to_IcoswISC30E3r5-nomask_trbilin.20240903.nc + cpl/gridmaps/ERA5r025/map_ERA5r025_to_IcoswISC30E3r5_esmfpatch.20240903.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_ERA5r025_traave.20240903.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_ERA5r025_traave.20240903.nc + + + + cpl/gridmaps/ERA5r025/map_ERA5r025_to_r05_traave.20240905.nc + cpl/gridmaps/ERA5r025/map_ERA5r025_to_r05_esmfbilin.20240907.nc + cpl/gridmaps/r05/map_r05_to_ERA5r025_traave.20240905.nc + cpl/gridmaps/r05/map_r05_to_ERA5r025_traave.20240905.nc + + + + cpl/gridmaps/ERA5r025/map_ERA5r025_to_r05_traave.20240905.nc + cpl/gridmaps/ERA5r025/map_ERA5r025_to_r05_esmfbilin.20240907.nc + + + + cpl/gridmaps/ERA5r025/map_ERA5r025_to_r025_traave.20240903.nc + cpl/gridmaps/ERA5r025/map_ERA5r025_to_r025_trbilin.20240903.nc + cpl/gridmaps/r025/map_r025_to_ERA5r025_traave.20240903.nc + cpl/gridmaps/r025/map_r025_to_ERA5r025_traave.20240903.nc + + + + cpl/gridmaps/ERA5r025/map_ERA5r025_to_r025_traave.20240903.nc + cpl/gridmaps/ERA5r025/map_ERA5r025_to_r025_trbilin.20240903.nc + + cpl/cpl6/map_T31_to_gx3v7_aave_da_090903.nc cpl/cpl6/map_T31_to_gx3v7_patch_090903.nc @@ -4902,6 +5105,10 @@ cpl/gridmaps/wQU225EC30to60E2r2/map_ne30pg2_TO_wQU225EC30to60E2r2_blin.20220222.nc + + cpl/gridmaps/wQU225Icos30E3r5/map_ne30pg2_to_wQU225Icos30E3r5_esmfbilin.20240910.nc + + cpl/gridmaps/wQU225EC30to60E2r2/map_TL319_TO_wQU225EC30to60E2r2_blin.20220602.nc @@ -4918,6 +5125,12 @@ cpl/gridmaps/wQU225EC30to60E2r2/map_EC30to60E2r2_TO_wQU225EC30to60E2r2_blin.20220222.nc + + cpl/gridmaps/wQU225Icos30E3r5/map_wQU225Icos30E3r5_to_IcoswISC30E3r5_esmfbilin.20240910.nc + cpl/gridmaps/wQU225Icos30E3r5/map_IcoswISC30E3r5_to_wQU225Icos30E3r5_esmfbilin.20240910.nc + cpl/gridmaps/wQU225Icos30E3r5/map_IcoswISC30E3r5_to_wQU225Icos30E3r5_esmfbilin.20240910.nc + + cpl/gridmaps/wQU225EC60to30/map_CFSv2_TO_wQU225EC60to30_blin.20210412.nc @@ -5147,6 +5360,10 @@ cpl/gridmaps/RRSwISC6to18E3r5/map_RRSwISC6to18E3r5_to_r05_traave.20240328.nc + + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_r05_traave.20240808.nc + + cpl/cpl6/map_EC30to60E2r2_to_r05_neareststod.220728.nc @@ -5262,6 +5479,11 @@ cpl/cpl6/map_rx1_to_RRSwISC6to18E3r5_cstmnn.r50e100.20240328.nc + + cpl/cpl6/map_rx1_to_SOwISC12to30E3r3_cstmnn.r150e300.20240808.nc + cpl/cpl6/map_rx1_to_SOwISC12to30E3r3_cstmnn.r150e300.20240808.nc + + cpl/cpl6/map_JRA025_to_oQU240wLI_cstmnn.r150e300.20240516.nc cpl/cpl6/map_JRA025_to_oQU240wLI_cstmnn.r150e300.20240516.nc @@ -5357,6 +5579,11 @@ cpl/cpl6/map_JRA025_to_RRSwISC6to18E3r5_cstmnn.r50e100.20240328.nc + + cpl/cpl6/map_JRA025_to_SOwISC12to30E3r3_cstmnn.r150e300.20240808.nc + cpl/cpl6/map_JRA025_to_SOwISC12to30E3r3_cstmnn.r150e300.20240808.nc + + cpl/cpl6/map_JRA025_to_oRRS18to6v3_smoothed.r50e100.220124.nc cpl/cpl6/map_JRA025_to_oRRS18to6v3_smoothed.r50e100.220124.nc @@ -5448,10 +5675,15 @@ - cpl/cpl6/map_r05_to_RRSwISC6to18E3r5.cstmnn.r250e1250_58NS.20240328.nc + cpl/cpl6/map_r05_to_RRSwISC6to18E3r5_cstmnn.r250e1250_58NS.20240328.nc cpl/cpl6/map_r05_to_RRSwISC6to18E3r5_cstmnn.r50e100.20240328.nc + + cpl/cpl6/map_r05_to_SOwISC12to30E3r3_r250e1250_58NS.cstmnn.20241120.nc + cpl/cpl6/map_r05_to_SOwISC12to30E3r3_cstmnn.r150e300.20240808.nc + + cpl/cpl6/map_r025_to_IcoswISC30E3r5_cstmnn.r150e300.20240401.nc cpl/cpl6/map_r025_to_IcoswISC30E3r5_cstmnn.r150e300.20240401.nc @@ -5549,15 +5781,15 @@ - cpl/gridmaps/oEC60to30v3/map_oEC60to30v3_to_mpas.aisgis20km_aave.190403.nc - cpl/gridmaps/oEC60to30v3/map_oEC60to30v3_to_mpas.aisgis20km_bilin.190403.nc + cpl/gridmaps/oEC60to30v3/map_oEC60to30v3_to_mpas.aisgis20km_aave.190403.nc + cpl/gridmaps/oEC60to30v3/map_oEC60to30v3_to_mpas.aisgis20km_bilin.190403.nc cpl/gridmaps/mpas.aisgis20km/map_mpas.aisgis20km_to_oEC60to30v3_aave.190403.nc cpl/gridmaps/mpas.aisgis20km/map_mpas.aisgis20km_to_oEC60to30v3_bilin.190403.nc - cpl/gridmaps/oEC60to30v3wLI/map_oEC60to30v3wLI_to_mpas.aisgis20km_aave.190713.nc - cpl/gridmaps/oEC60to30v3wLI/map_oEC60to30v3wLI_to_mpas.aisgis20km_bilin.190713.nc + cpl/gridmaps/oEC60to30v3wLI/map_oEC60to30v3wLI_to_mpas.aisgis20km_aave.190713.nc + cpl/gridmaps/oEC60to30v3wLI/map_oEC60to30v3wLI_to_mpas.aisgis20km_bilin.190713.nc cpl/gridmaps/mpas.aisgis20km/map_mpas.aisgis20km_to_oEC60to30v3wLI_aave.190713.nc cpl/gridmaps/mpas.aisgis20km/map_mpas.aisgis20km_to_oEC60to30v3wLI_bilin.190713.nc @@ -5574,8 +5806,8 @@ - cpl/gridmaps/oEC60to30v3/map_oEC60to30v3_to_mpas.gis20km_aave.181115.nc - cpl/gridmaps/oEC60to30v3/map_oEC60to30v3_to_mpas.gis20km_bilin.181115.nc + cpl/gridmaps/oEC60to30v3/map_oEC60to30v3_to_mpas.gis20km_aave.181115.nc + cpl/gridmaps/oEC60to30v3/map_oEC60to30v3_to_mpas.gis20km_bilin.181115.nc cpl/gridmaps/mpas.gis20km/map_mpas.gis20km_to_oEC60to30v3_aave.181115.nc cpl/gridmaps/mpas.gis20km/map_mpas.gis20km_to_oEC60to30v3_aave.181115.nc @@ -5601,9 +5833,21 @@ cpl/gridmaps/mpas.gis20km/map_gis20km_to_TL319_traave.20240404.nc + + cpl/gridmaps/oQU240wLI/map_oQU240wLI_to_gis20km_esmfaave.20240919.nc + cpl/gridmaps/oQU240wLI/map_oQU240wLI_to_gis20km_esmfbilin.20240919.nc + cpl/gridmaps/oQU240wLI/map_oQU240wLI_to_gis20km_esmfneareststod.20240919.deeperThan300m.nc + cpl/gridmaps/mpas.gis20km/map_gis20km_to_oQU240wLI_esmfaave.20240919.nc + cpl/gridmaps/mpas.gis20km/map_gis20km_to_oQU240wLI_esmfaave.20240919.nc + cpl/gridmaps/mpas.gis20km/map_gis20km_to_oQU240wLI_esmfaave.20240919.nc + cpl/gridmaps/mpas.gis20km/map_gis20km_to_oQU240wLI_esmfaave.20240919.nc + cpl/gridmaps/mpas.gis20km/map_gis20km_to_oQU240wLI_esmfaave.20240919.nc + cpl/gridmaps/mpas.gis20km/map_gis20km_to_oQU240wLI_esmfaave.20240919.nc + + - cpl/gridmaps/EC30to60E2r2/map_EC30to60E2r2_to_gis20km_aave.230510.nc - cpl/gridmaps/EC30to60E2r2/map_EC30to60E2r2_to_gis20km_bilin.230510.nc + cpl/gridmaps/EC30to60E2r2/map_EC30to60E2r2_to_gis20km_aave.230510.nc + cpl/gridmaps/EC30to60E2r2/map_EC30to60E2r2_to_gis20km_bilin.230510.nc cpl/gridmaps/mpas.gis20km/map_gis20km_to_EC30to60E2r2_aave.230510.nc cpl/gridmaps/mpas.gis20km/map_gis20km_to_EC30to60E2r2_aave.230510.nc cpl/gridmaps/mpas.gis20km/map_gis20km_to_EC30to60E2r2_aave.230510.nc @@ -5613,8 +5857,9 @@ - cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_gis20km_esmfaave.20240403.nc - cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_gis20km_esmfbilin.20240403.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_gis20km_esmfaave.20240403.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_gis20km_esmfbilin.20240403.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_gis20km_esmfneareststod.20240422.deeperThan300m.nc cpl/gridmaps/mpas.gis20km/map_gis20km_to_IcoswISC30E3r5_esmfaave.20240403.nc cpl/gridmaps/mpas.gis20km/map_gis20km_to_IcoswISC30E3r5_esmfaave.20240403.nc cpl/gridmaps/mpas.gis20km/map_gis20km_to_IcoswISC30E3r5_esmfaave.20240403.nc @@ -5649,8 +5894,8 @@ - cpl/gridmaps/oEC60to30v3/map_oEC60to30v3_to_gis1to10km_aave.200602.nc - cpl/gridmaps/oEC60to30v3/map_oEC60to30v3_to_gis1to10km_bilin.200602.nc + cpl/gridmaps/oEC60to30v3/map_oEC60to30v3_to_gis1to10km_aave.200602.nc + cpl/gridmaps/oEC60to30v3/map_oEC60to30v3_to_gis1to10km_bilin.200602.nc cpl/gridmaps/mpas.gis1to10km/map_gis1to10km_to_oEC60to30v3_aave.200602.nc cpl/gridmaps/mpas.gis1to10km/map_gis1to10km_to_oEC60to30v3_aave.200602.nc cpl/gridmaps/mpas.gis1to10km/map_gis1to10km_to_oEC60to30v3_aave.200602.nc @@ -5660,8 +5905,8 @@ - cpl/gridmaps/EC30to60E2r2/map_EC30to60E2r2_to_gis1to10km_aave.210304.nc - cpl/gridmaps/EC30to60E2r2/map_EC30to60E2r2_to_gis1to10km_bilin.210304.nc + cpl/gridmaps/EC30to60E2r2/map_EC30to60E2r2_to_gis1to10km_aave.210304.nc + cpl/gridmaps/EC30to60E2r2/map_EC30to60E2r2_to_gis1to10km_bilin.210304.nc cpl/gridmaps/mpas.gis1to10km/map_gis1to10km_to_EC30to60E2r2_aave.210304.nc cpl/gridmaps/mpas.gis1to10km/map_gis1to10km_to_EC30to60E2r2_aave.210304.nc cpl/gridmaps/mpas.gis1to10km/map_gis1to10km_to_EC30to60E2r2_aave.210304.nc @@ -5681,6 +5926,13 @@ cpl/gridmaps/mpas.gis1to10km/map_gis1to10kmR2_to_r05_traave.20240403.nc + + cpl/gridmaps/r025/map_r025_to_gis1to10kmR2_traave.20240903.nc + cpl/gridmaps/r025/map_r025_to_gis1to10kmR2_trbilin.20240903.nc + cpl/gridmaps/mpas.gis1to10km/map_gis1to10kmR2_to_r025_traave.20240903.nc + cpl/gridmaps/mpas.gis1to10km/map_gis1to10kmR2_to_r025_traave.20240903.nc + + cpl/gridmaps/ne30pg2/map_ne30pg2_to_gis1to10kmR2_traave.20240403.nc cpl/gridmaps/ne30pg2/map_ne30pg2_to_gis1to10kmR2_trbilin.20240403.nc @@ -5696,8 +5948,8 @@ - cpl/gridmaps/EC30to60E2r2/map_EC30to60E2r2_to_gis1to10r02_aave.230725.nc - cpl/gridmaps/EC30to60E2r2/map_EC30to60E2r2_to_gis1to10r02_bilin.230725.nc + cpl/gridmaps/EC30to60E2r2/map_EC30to60E2r2_to_gis1to10r02_aave.230725.nc + cpl/gridmaps/EC30to60E2r2/map_EC30to60E2r2_to_gis1to10r02_bilin.230725.nc cpl/gridmaps/mpas.gis1to10km/map_gis1to10r02_to_EC30to60E2r2_aave.230725.nc cpl/gridmaps/mpas.gis1to10km/map_gis1to10r02_to_EC30to60E2r2_aave.230725.nc cpl/gridmaps/mpas.gis1to10km/map_gis1to10r02_to_EC30to60E2r2_aave.230725.nc @@ -5707,8 +5959,9 @@ - cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_gis1to10kmR2_esmfaave.20240403.nc - cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_gis1to10kmR2_esmfbilin.20240403.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_gis1to10kmR2_esmfaave.20240403.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_gis1to10kmR2_esmfbilin.20240403.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_gis1to10kmR2_esmfneareststod.20240422.deeperThan300m.nc cpl/gridmaps/mpas.gis1to10km/map_gis1to10kmR2_to_IcoswISC30E3r5_esmfaave.20240403.nc cpl/gridmaps/mpas.gis1to10km/map_gis1to10kmR2_to_IcoswISC30E3r5_esmfaave.20240403.nc cpl/gridmaps/mpas.gis1to10km/map_gis1to10kmR2_to_IcoswISC30E3r5_esmfaave.20240403.nc @@ -5736,8 +5989,8 @@ - cpl/gridmaps/oQU240wLI/map_oQU240wLI-nomask_to_ais20km_esmfaave.20240509.nc - cpl/gridmaps/oQU240wLI/map_oQU240wLI-nomask_to_ais20km_esmfbilin.20240509.nc + cpl/gridmaps/oQU240wLI/map_oQU240wLI-nomask_to_ais20km_esmfaave.20240509.nc + cpl/gridmaps/oQU240wLI/map_oQU240wLI-nomask_to_ais20km_esmfbilin.20240509.nc cpl/gridmaps/mpas.ais20km/map_ais20km_to_oQU240wLI-nomask_esmfaave.20240509.nc cpl/gridmaps/mpas.ais20km/map_ais20km_to_oQU240wLI-nomask_esmfbilin.20240509.nc cpl/gridmaps/mpas.ais20km/map_ais20km_to_oQU240wLI-nomask_esmfaave.20240509.nc @@ -5765,8 +6018,8 @@ - cpl/gridmaps/oQU240wLI/map_oQU240wLI-nomask_to_ais8to30_esmfaave.20240701.nc - cpl/gridmaps/oQU240wLI/map_oQU240wLI-nomask_to_ais8to30_esmfbilin.20240701.nc + cpl/gridmaps/oQU240wLI/map_oQU240wLI-nomask_to_ais8to30_esmfaave.20240701.nc + cpl/gridmaps/oQU240wLI/map_oQU240wLI-nomask_to_ais8to30_esmfbilin.20240701.nc cpl/gridmaps/mpas.ais8to30km/map_ais8to30_to_oQU240wLI-nomask_esmfaave.20240701.nc cpl/gridmaps/mpas.ais8to30km/map_ais8to30_to_oQU240wLI-nomask_esmfbilin.20240701.nc cpl/gridmaps/mpas.ais8to30km/map_ais8to30_to_oQU240wLI-nomask_esmfaave.20240701.nc @@ -5776,8 +6029,8 @@ - cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5-nomask_to_ais8to30_esmfaave.20240701.nc - cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5-nomask_to_ais8to30_esmfbilin.20240701.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5-nomask_to_ais8to30_esmfaave.20240701.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5-nomask_to_ais8to30_esmfbilin.20240701.nc cpl/gridmaps/mpas.ais8to30km/map_ais8to30_to_IcoswISC30E3r5-nomask_esmfaave.20240701.nc cpl/gridmaps/mpas.ais8to30km/map_ais8to30_to_IcoswISC30E3r5-nomask_esmfbilin.20240701.nc cpl/gridmaps/mpas.ais8to30km/map_ais8to30_to_IcoswISC30E3r5-nomask_esmfaave.20240701.nc @@ -5805,8 +6058,8 @@ - cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5-nomask_to_ais4to20_esmfaave.20240701.nc - cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5-nomask_to_ais4to20_esmfbilin.20240701.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5-nomask_to_ais4to20_esmfaave.20240701.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5-nomask_to_ais4to20_esmfbilin.20240701.nc cpl/gridmaps/mpas.ais4to20km/map_ais4to20_to_IcoswISC30E3r5-nomask_esmfaave.20240701.nc cpl/gridmaps/mpas.ais4to20km/map_ais4to20_to_IcoswISC30E3r5-nomask_esmfbilin.20240701.nc cpl/gridmaps/mpas.ais4to20km/map_ais4to20_to_IcoswISC30E3r5-nomask_esmfaave.20240701.nc @@ -5923,11 +6176,11 @@ cpl/gridmaps/mpas.ais20km/map_ais20km_to_oQU240_aave.151209.nc - cpl/gridmaps/oQU240/map_oQU240_to_ais20km_aave.151209.nc + cpl/gridmaps/oQU240/map_oQU240_to_ais20km_aave.151209.nc cpl/gridmaps/mpas.ais20km/map_ais20km_to_oQU240_nearestdtos.151209.nc cpl/gridmaps/mpas.ais20km/map_ais20km_to_oQU240_nearestdtos.151209.nc cpl/gridmaps/mpas.ais20km/map_ais20km_to_oQU240_nearestdtos.151209.nc - cpl/gridmaps/oQU240/map_oQU240_to_ais20km_nearestdtos.151209.nc + cpl/gridmaps/oQU240/map_oQU240_to_ais20km_nearestdtos.151209.nc @@ -5937,8 +6190,8 @@ cpl/gridmaps/mpas.ais20km/map_ais20km_to_oQU120_nearestdtos.160331.nc cpl/gridmaps/mpas.ais20km/map_ais20km_to_oQU120_nearestdtos.160331.nc cpl/gridmaps/mpas.ais20km/map_ais20km_to_oQU120_nearestdtos.160331.nc - cpl/gridmaps/oQU120/map_oQU120_to_ais20km_aave.160331.nc - cpl/gridmaps/oQU120/map_oQU120_to_ais20km_neareststod.160331.nc + cpl/gridmaps/oQU120/map_oQU120_to_ais20km_aave.160331.nc + cpl/gridmaps/oQU120/map_oQU120_to_ais20km_neareststod.160331.nc @@ -5948,8 +6201,8 @@ cpl/gridmaps/mpas.ais20km/map_ais20km_to_oEC60to30v3wLI_nomask_nearestdtos.190207.nc cpl/gridmaps/mpas.ais20km/map_ais20km_to_oEC60to30v3wLI_nomask_nearestdtos.190207.nc cpl/gridmaps/mpas.ais20km/map_ais20km_to_oEC60to30v3wLI_nomask_nearestdtos.190207.nc - cpl/gridmaps/oEC60to30v3wLI/map_oEC60to30v3wLI_nomask_to_ais20km_aave.190207.nc - cpl/gridmaps/oEC60to30v3wLI/map_oEC60to30v3wLI_nomask_to_ais20km_neareststod.190207.nc + cpl/gridmaps/oEC60to30v3wLI/map_oEC60to30v3wLI_nomask_to_ais20km_aave.190207.nc + cpl/gridmaps/oEC60to30v3wLI/map_oEC60to30v3wLI_nomask_to_ais20km_neareststod.190207.nc diff --git a/cime_config/machines/Depends.crayclanggpu.cmake b/cime_config/machines/Depends.crayclanggpu.cmake index eaff237a27fc..3b0881ccd04e 100644 --- a/cime_config/machines/Depends.crayclanggpu.cmake +++ b/cime_config/machines/Depends.crayclanggpu.cmake @@ -4,6 +4,8 @@ list(APPEND NOOPT_FILES elm/src/data_types/VegetationDataType.F90 elm/src/biogeochem/CNNitrogenFluxType.F90 elm/src/biogeochem/CNCarbonFluxType.F90 + mosart/src/wrm/WRM_subw_IO_mod.F90 + mosart/src/riverroute/RtmMod.F90 ) # Files added below to mitigate excessive compilation times diff --git a/cime_config/machines/Depends.muller-cpu.gnu.cmake b/cime_config/machines/Depends.muller-cpu.gnu.cmake index 5c7331f979a9..53f8b536651e 100644 --- a/cime_config/machines/Depends.muller-cpu.gnu.cmake +++ b/cime_config/machines/Depends.muller-cpu.gnu.cmake @@ -7,23 +7,3 @@ if (NOT DEBUG) e3sm_deoptimize_file("${ITEM}") endforeach() endif() - -# On pm-cpu (and muller-cpu), with gcc-native/12.3, we see hang with DEBUG runs of certain tests. -# https://github.com/E3SM-Project/E3SM/issues/6516 -# Currently, we have pm-cpu using gcc/12.2.0 which does not have this issue, but using muller-cpu to test 12.3 -# Turning off -O0 for these 2 files (by adding -O) at least avoids hang and will produce FPE in HOMME code -if (CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER_EQUAL 12.3) - if (DEBUG) - - set(ADJUST - eam/src/dynamics/se/inidat.F90 - eam/src/dynamics/se/dyn_comp.F90 - ) - - foreach(ITEM IN LISTS ADJUST) - e3sm_add_flags("${ITEM}" "-O") - #e3sm_add_flags("${ITEM}" "-DNDEBUG -O") - endforeach() - - endif() -endif() diff --git a/cime_config/machines/cmake_macros/amdclanggpu_frontier.cmake b/cime_config/machines/cmake_macros/amdclanggpu_frontier.cmake index 4412ea0de7b3..1deebdac85dc 100644 --- a/cime_config/machines/cmake_macros/amdclanggpu_frontier.cmake +++ b/cime_config/machines/cmake_macros/amdclanggpu_frontier.cmake @@ -13,8 +13,6 @@ string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2") string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") -string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") - string(APPEND CMAKE_CXX_FLAGS " --offload-arch=gfx90a") string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa") string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/opt/cray/pe/gcc/12.2.0/snos/lib64 -lgfortran -lstdc++") diff --git a/cime_config/machines/cmake_macros/crayclanggpu_frontier.cmake b/cime_config/machines/cmake_macros/crayclanggpu_frontier.cmake index a37ccde439e5..494638443477 100644 --- a/cime_config/machines/cmake_macros/crayclanggpu_frontier.cmake +++ b/cime_config/machines/cmake_macros/crayclanggpu_frontier.cmake @@ -1,6 +1,5 @@ set(MPICC "cc") set(MPICXX "mpicxx") -#set(MPICXX "CC") set(MPIFC "ftn") set(SCC "cc") set(SCXX "hipcc") @@ -34,7 +33,7 @@ set(HAS_F2008_CONTIGUOUS "TRUE") # -Wl,--allow-shlib-undefined was added to address rocm 5.4.3 Fortran linker issue: # /opt/rocm-5.4.3/lib/libhsa-runtime64.so.1: undefined reference to `std::condition_variable::wait(std::unique_lock&)@GLIBCXX_3.4.30' # AMD started building with GCC 12.2.0, which brings in a GLIBCXX symbol that isn't in CCE's default GCC toolchain. -#string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--allow-multiple-definition -Wl,--allow-shlib-undefined") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--allow-shlib-undefined -Wl,--allow-multiple-definition") # Switching to O3 for performance benchmarking # Will revisit any failing tests @@ -52,8 +51,6 @@ endif() # https://github.com/E3SM-Project/E3SM/pull/5208 string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero -em -ef -hnoacc") -string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") - string(APPEND CMAKE_CXX_FLAGS " --offload-arch=gfx90a") string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa") string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64") diff --git a/cime_config/machines/cmake_macros/gnu_chicoma-cpu.cmake b/cime_config/machines/cmake_macros/gnu_chicoma-cpu.cmake index a6148451eb76..a6c13942620f 100644 --- a/cime_config/machines/cmake_macros/gnu_chicoma-cpu.cmake +++ b/cime_config/machines/cmake_macros/gnu_chicoma-cpu.cmake @@ -5,6 +5,7 @@ endif() set(PIO_FILESYSTEM_HINTS "lustre") string(APPEND CMAKE_C_FLAGS_RELEASE " -O2 -g") string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2 -g") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--enable-new-dtags") set(MPICC "cc") set(MPICXX "CC") set(MPIFC "ftn") diff --git a/cime_config/machines/cmake_macros/gnu_chicoma-gpu.cmake b/cime_config/machines/cmake_macros/gnu_chicoma-gpu.cmake index 807c7d0211eb..a6c13942620f 100644 --- a/cime_config/machines/cmake_macros/gnu_chicoma-gpu.cmake +++ b/cime_config/machines/cmake_macros/gnu_chicoma-gpu.cmake @@ -2,15 +2,10 @@ string(APPEND CONFIG_ARGS " --host=cray") if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY") endif() -string(APPEND SLIBS " -lblas -llapack") -set(CXX_LINKER "FORTRAN") -if (NOT DEBUG) - string(APPEND CFLAGS " -O2 -g") -endif() -if (NOT DEBUG) - string(APPEND FFLAGS " -O2 -g") -endif() -string(APPEND CXX_LIBS " -lstdc++") +set(PIO_FILESYSTEM_HINTS "lustre") +string(APPEND CMAKE_C_FLAGS_RELEASE " -O2 -g") +string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2 -g") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--enable-new-dtags") set(MPICC "cc") set(MPICXX "CC") set(MPIFC "ftn") diff --git a/cime_config/machines/cmake_macros/gnugpu_frontier.cmake b/cime_config/machines/cmake_macros/gnugpu_frontier.cmake index 6ca4b83d9c2f..7a29a5ca1546 100644 --- a/cime_config/machines/cmake_macros/gnugpu_frontier.cmake +++ b/cime_config/machines/cmake_macros/gnugpu_frontier.cmake @@ -14,7 +14,6 @@ string(APPEND CMAKE_Fortran_FLAGS " -Wno-implicit-interface") string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2") string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") -string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") string(APPEND CMAKE_CXX_FLAGS " --offload-arch=gfx90a") string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa") diff --git a/cime_config/machines/cmake_macros/intel_dane.cmake b/cime_config/machines/cmake_macros/intel_dane.cmake index 8091325c6cec..ef25a97b300e 100644 --- a/cime_config/machines/cmake_macros/intel_dane.cmake +++ b/cime_config/machines/cmake_macros/intel_dane.cmake @@ -1,4 +1,10 @@ string(APPEND CPPDEFS " -DNO_SHR_VMATH -DCNL") string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -check all -ftrapuv") string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/usr/tce/packages/gcc/gcc-10.3.1-magic/lib/gcc/x86_64-redhat-linux/10/") + +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/hdf5-1.10.7-766kapalbrdntu2pcgdgbhg2ch26gsuv/lib") +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/netcdf-c-4.4.1.1-2uznnlwgiezxute6iyqzqjrpolokeaib/lib") +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/netcdf-fortran-4.4.4-itpstyordbern7vlulmlnt47eeeokzfp/lib") +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/parallel-netcdf-1.11.0-26sxm4mormsglmhi24poix7sugbigkck/lib") + set(KOKKOS_OPTIONS "--with-serial --ldflags='-L/usr/tce/packages/gcc/gcc-10.3.1-magic/lib/gcc/x86_64-redhat-linux/10/'") diff --git a/cime_config/machines/cmake_macros/intel_ruby.cmake b/cime_config/machines/cmake_macros/intel_ruby.cmake index 8091325c6cec..e874bfb7eaf6 100644 --- a/cime_config/machines/cmake_macros/intel_ruby.cmake +++ b/cime_config/machines/cmake_macros/intel_ruby.cmake @@ -1,4 +1,10 @@ string(APPEND CPPDEFS " -DNO_SHR_VMATH -DCNL") string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -check all -ftrapuv") string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/usr/tce/packages/gcc/gcc-10.3.1-magic/lib/gcc/x86_64-redhat-linux/10/") + +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/hdf5-1.10.7-ewjpbjdhjgjzrzjcvwyjyuulaesbsjhg/lib") +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/netcdf-c-4.4.1.1-vaxofekwvnvngh7wptmzkwdb7tkzvesn/lib") +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/netcdf-fortran-4.4.4-3pzbx2unddhladhubaahhhysjmprzqi2/lib") +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/parallel-netcdf-1.11.0-tzgdalakmem7tod6cruhqyeackeix5q5/lib") + set(KOKKOS_OPTIONS "--with-serial --ldflags='-L/usr/tce/packages/gcc/gcc-10.3.1-magic/lib/gcc/x86_64-redhat-linux/10/'") diff --git a/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake b/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake index c6afa7c2329c..e5c486f216e9 100644 --- a/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake +++ b/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake @@ -3,7 +3,10 @@ string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_c if (compile_threaded) string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64") endif() + string(APPEND KOKKOS_OPTIONS " -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_SERIAL=On -DKokkos_ARCH_INTEL_PVC=On -DKokkos_ENABLE_SYCL=On -DKokkos_ENABLE_EXPLICIT_INSTANTIATION=Off") string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 -Xsycl-target-backend \"-device 12.60.7\"") +string(APPEND OMEGA_SYCL_EXE_LINKER_FLAGS " -Xsycl-target-backend \"-device 12.60.7\" ") +string(APPEND CMAKE_CXX_FLAGS " -Xclang -fsycl-allow-virtual-functions") set(SCREAM_MPI_ON_DEVICE OFF CACHE STRING "") diff --git a/cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot.cmake b/cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot.cmake index 6835515164f0..1ad0d6e6b61a 100644 --- a/cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot.cmake +++ b/cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot.cmake @@ -3,5 +3,6 @@ string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_c if (compile_threaded) string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64") endif() -string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 -Xsycl-target-backend \"-device 12.60.7\"") +string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 ") +string(APPEND OMEGA_SYCL_EXE_LINKER_FLAGS " -Xsycl-target-backend \"-device 12.60.7\" ") string(APPEND CMAKE_CXX_FLAGS " -Xclang -fsycl-allow-virtual-functions") diff --git a/cime_config/machines/config_batch.xml b/cime_config/machines/config_batch.xml index 3abac928ac2a..dcdf52178ef1 100644 --- a/cime_config/machines/config_batch.xml +++ b/cime_config/machines/config_batch.xml @@ -237,33 +237,17 @@ - - squeue - sbatch - scancel - #SBATCH - (\d+)$ - --dependency=afterok:jobid - --dependency=afterany:jobid - : - %H:%M:%S - --mail-user - --mail-type - none, all, begin, end, fail - - --export=ALL - -p {{ job_queue }} - -J {{ job_id }} - -N {{ num_nodes }} - -n {{ total_tasks }} - -t {{ job_wallclock_time }} - -o {{ job_id }}.out - -e {{ job_id }}.err - -A {{ project }} - + - pbatch - pdebug + pbatch + pdebug + + + + + + pbatch + pdebug @@ -464,6 +448,7 @@ regular preempt shared + overrun debug @@ -518,10 +503,11 @@ regular preempt shared + overrun debug - + --constraint=cpu diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 07308c066b8b..cbd857ee1547 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -190,6 +190,7 @@ module + cpe cray-hdf5-parallel cray-netcdf-hdf5parallel cray-parallel-netcdf @@ -218,34 +219,34 @@ PrgEnv-gnu/8.5.0 - gcc/12.2.0 - cray-libsci/23.02.1.1 + gcc-native/12.3 + cray-libsci/23.12.5 - PrgEnv-intel/8.3.3 - intel/2023.1.0 + PrgEnv-intel/8.5.0 + intel/2023.2.0 PrgEnv-nvidia - nvidia/22.7 - cray-libsci/23.02.1.1 + nvidia/24.5 + cray-libsci/23.12.5 PrgEnv-aocc - aocc/4.0.0 - cray-libsci/23.02.1.1 + aocc/4.1.0 + cray-libsci/23.12.5 craype-accel-host - craype/2.7.20 - cray-mpich/8.1.25 - cray-hdf5-parallel/1.12.2.3 - cray-netcdf-hdf5parallel/4.9.0.3 - cray-parallel-netcdf/1.12.3.3 + craype/2.7.30 + cray-mpich/8.1.28 + cray-hdf5-parallel/1.12.2.9 + cray-netcdf-hdf5parallel/4.9.0.9 + cray-parallel-netcdf/1.12.3.9 cmake/3.24.3 evp-patch @@ -265,7 +266,7 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch - software + kdreg2 MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} @@ -355,6 +356,7 @@ module + cpe cray-hdf5-parallel cray-netcdf-hdf5parallel cray-parallel-netcdf @@ -365,13 +367,14 @@ PrgEnv-nvidia PrgEnv-cray PrgEnv-aocc + gcc-native intel intel-oneapi nvidia aocc cudatoolkit - cray-libsci climate-utils + cray-libsci matlab craype-accel-nvidia80 craype-accel-host @@ -423,6 +426,7 @@ $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 + 0.20 1 @@ -433,6 +437,7 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch + kdreg2 MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} @@ -578,12 +583,11 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch - software + kdreg2 MPI_Bcast - $SHELL{if [ -z "$Albany_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/albany-e3sm-serial-release-gcc; else echo "$Albany_ROOT"; fi} - $SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/trilinos-e3sm-serial-release-gcc; else echo "$Trilinos_ROOT"; fi} $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} + 4000MB $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/intel-2023.1.0; else echo "$ADIOS2_ROOT"; fi} @@ -591,6 +595,8 @@ $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/gcc-11.2.0; else echo "$ADIOS2_ROOT"; fi} Generic + $SHELL{if [ -z "$Albany_ROOT" ]; then echo /global/common/software/e3sm/albany/2024.03.26/gcc/11.2.0; else echo "$Albany_ROOT"; fi} + $SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /global/common/software/e3sm/trilinos/15.1.1/gcc/11.2.0; else echo "$Trilinos_ROOT"; fi} $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/nvidia-22.7; else echo "$ADIOS2_ROOT"; fi} @@ -606,6 +612,13 @@ $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/aocc-4.0.0; else echo "$ADIOS2_ROOT"; fi} + + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/intel; else echo "$MOAB_ROOT"; fi} + + + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/gnu; else echo "$MOAB_ROOT"; fi} + + -1 @@ -672,13 +685,14 @@ PrgEnv-nvidia PrgEnv-cray PrgEnv-aocc + gcc-native intel intel-oneapi nvidia aocc cudatoolkit - cray-libsci climate-utils + cray-libsci matlab craype-accel-nvidia80 craype-accel-host @@ -730,6 +744,7 @@ $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 + 0.20 1 @@ -740,6 +755,7 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch + kdreg2 MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} @@ -750,6 +766,9 @@ 1 + + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/gnugpu ; else echo "$MOAB_ROOT"; fi} + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/gcc-11.2.0; else echo "$ADIOS2_ROOT"; fi} @@ -807,6 +826,7 @@ module + cpe cray-hdf5-parallel cray-netcdf-hdf5parallel cray-parallel-netcdf @@ -834,36 +854,35 @@ - PrgEnv-gnu - gcc-native - cray-libsci + PrgEnv-gnu/8.5.0 + gcc-native/13.2 + cray-libsci/24.03.0 - PrgEnv-intel - intel + PrgEnv-intel/8.5.0 + intel/2024.1.0 PrgEnv-nvidia nvidia/24.5 - cray-libsci + cray-libsci/24.03.0 PrgEnv-aocc - aocc/4.0.1 - cray-libsci + aocc/4.1.0 + cray-libsci/24.03.0 craype-accel-host - cray-libsci - craype/2.7.30 - cray-mpich/8.1.28 - cray-hdf5-parallel/1.12.2.9 - cray-netcdf-hdf5parallel/4.9.0.9 - cray-parallel-netcdf/1.12.3.9 + craype/2.7.31.11 + cray-mpich/8.1.29 + cray-hdf5-parallel/1.12.2.11 + cray-netcdf-hdf5parallel/4.9.0.11 + cray-parallel-netcdf/1.12.3.11 cmake/3.24.3 @@ -871,6 +890,7 @@ $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 + 0.20 1 @@ -881,16 +901,48 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch - software + kdreg2 MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} + 4000MB + + + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/intel-2023.1.0; else echo "$ADIOS2_ROOT"; fi} + + + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/gcc-11.2.0; else echo "$ADIOS2_ROOT"; fi} + Generic + $SHELL{if [ -z "$Albany_ROOT" ]; then echo /global/common/software/e3sm/albany/2024.03.26/gcc/11.2.0; else echo "$Albany_ROOT"; fi} + $SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /global/common/software/e3sm/trilinos/15.1.1/gcc/11.2.0; else echo "$Trilinos_ROOT"; fi} + + + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/nvidia-22.7; else echo "$ADIOS2_ROOT"; fi} + + + $SHELL{if [ -z "$BLAS_ROOT" ]; then echo $NVIDIA_PATH/compilers; else echo "$BLAS_ROOT"; fi} + $SHELL{if [ -z "$LAPACK_ROOT" ]; then echo $NVIDIA_PATH/compilers; else echo "$LAPACK_ROOT"; fi} + NVHPC + + + Intel10_64_dyn + + + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/aocc-4.0.0; else echo "$ADIOS2_ROOT"; fi} + + + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/intel; else echo "$MOAB_ROOT"; fi} + + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/gnu; else echo "$MOAB_ROOT"; fi} + + -1 + Spock. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL. .*spock.* @@ -1017,6 +1069,7 @@ /usr/share/lmod/lmod/libexec/lmod python + Core Core/24.00 PrgEnv-cray PrgEnv-cray/8.3.3 cce cce/15.0.1 @@ -1029,14 +1082,22 @@ + Core Core/24.00 PrgEnv-cray PrgEnv-amd/8.3.3 amd amd/5.4.0 + + + + Core Core/24.00 PrgEnv-cray PrgEnv-gnu/8.3.3 gcc gcc/12.2.0 @@ -1047,10 +1108,11 @@ cray-python/3.9.13.1 cray-libsci - subversion/1.14.1 - git/2.36.1 cmake/3.21.3 - zlib/1.2.11 + subversion + git + zlib + libfabric/1.15.2.0 cray-hdf5-parallel/1.12.2.1 cray-netcdf-hdf5parallel/4.9.0.1 cray-parallel-netcdf/1.12.3.1 @@ -1084,9 +1146,6 @@ spread threads - @@ -1160,10 +1218,11 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss + Core/24.00 cpe/22.12 craype-accel-amd-gfx90a rocm/5.4.0 - libunwind/1.6.2 + libunwind/1.5.0 cce/15.0.1 libfabric/1.15.2.0 craype/2.7.20 @@ -1632,6 +1691,49 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss + + Huge Linux workstation for Sandia climate scientists + ^[a-fA-F0-9]{12}$ + LINUX + proxy.sandia.gov:80 + gnu + openmpi + /projects/e3sm/scratch + /projects/e3sm/inputdata + /projects/e3sm/inputdata/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + /projects/e3sm/baselines/ghci-snl-cpu/$COMPILER + /projects/e3sm/cprnc/cprnc + 32 + e3sm_developer + none + lbertag at sandia dot gov + 32 + 32 + + mpirun + + --bind-to core + -np {{ total_tasks }} + + + + $CIME_OUTPUT_ROOT/$CASE/run + $CIME_OUTPUT_ROOT/$CASE/bld + 0.1 + 0 + + $ENV{NETCDF_C_ROOT} + $ENV{NETCDF_FORTRAN_ROOT} + $ENV{PARALLEL_NETCDF_ROOT} + 64M + spread + threads + Generic + 4000MB + + + Sandia GPU testbed weaver @@ -1932,7 +2034,8 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss /nfs/gce/projects/climate/software/perl5/lib/perl5 - $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/adios2/2.9.1/mpich-4.1.2/gcc-12.1.0; else echo "$ADIOS2_ROOT"; fi} + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/adios2/2.10.1/mpich-4.1.2/gcc-12.1.0; else echo "$ADIOS2_ROOT"; fi} + $SHELL{if [ -z "$BLOSC2_ROOT" ]; then echo /nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/c-blosc2/2.15.1/gcc-12.1.0; else echo "$BLOSC2_ROOT"; fi} @@ -2354,17 +2457,15 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss openmpi/4.1.6-2mm63n2 hdf5/1.10.7-4cghwvq - netcdf-c/4.4.1-a4hji6e - netcdf-cxx/4.2-ldoxr43 - netcdf-fortran/4.4.4-husened + netcdf-c/4.7.4-4qjdadt + netcdf-fortran/4.5.3-qozrykr parallel-netcdf/1.11.0-icrpxty intel-mpi/2019.9.304-tkzvizk - hdf5/1.8.16-se4xyo7 - netcdf-c/4.4.1-qvxyzq2 - netcdf-cxx/4.2-binixgj - netcdf-fortran/4.4.4-rdxohvp + hdf5/1.10.7-wczt56s + netcdf-c/4.7.4-ba6agmb + netcdf-fortran/4.5.3-5lvy5p4 parallel-netcdf/1.11.0-b74wv4m @@ -2374,17 +2475,19 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss openmpi/4.1.6-ggebj5o hdf5/1.10.7-ol6xuae - netcdf-c/4.4.1-2njo6xx - netcdf-cxx/4.2-7pdzqua - netcdf-fortran/4.4.4-52c6oqi + netcdf-c/4.7.4-pfocec2 + netcdf-fortran/4.5.3-va3hoor parallel-netcdf/1.11.0-d7h4ysd + gcc/11.2.0-bgddrif + intel-oneapi-mkl/2022.1.0-w4kgsn4 + gcc/9.2.0-ugetvbp + intel-mkl/2020.4.304-n3b5fye intel-mpi/2019.9.304-jdih7h5 hdf5/1.8.16-dtbpce3 - netcdf-c/4.4.1-zcoa44z - netcdf-cxx/4.2-ayxg4c7 - netcdf-fortran/4.4.4-2lfr2lr + netcdf-c/4.7.4-seagl7g + netcdf-fortran/4.5.3-ova6t37 parallel-netcdf/1.11.0-ifdodru @@ -2754,7 +2857,8 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss /lcrc/group/e3sm/soft/improv/pnetcdf/1.12.3/gcc-12.3.0/openmpi-4.1.6 /lcrc/group/e3sm/soft/improv/pnetcdf/1.12.3/gcc-12.3.0/openmpi-4.1.6/bin:/lcrc/group/e3sm/soft/improv/netcdf-fortran/4.6.1b/gcc-12.3.0/openmpi-4.1.6/bin:/lcrc/group/e3sm/soft/improv/netcdf-c/4.9.2b/gcc-12.3.0/openmpi-4.1.6/bin:/lcrc/group/e3sm/soft/improv/openmpi/4.1.6/gcc-12.3.0/bin:/lcrc/group/e3sm/soft/perl/improv/bin:$ENV{PATH} $SHELL{lp=/lcrc/group/e3sm/soft/improv/netlib-lapack/3.12.0/gcc-12.3.0:/lcrc/group/e3sm/soft/improv/pnetcdf/1.12.3/gcc-12.3.0/openmpi-4.1.6/lib:/lcrc/group/e3sm/soft/improv/netcdf-fortran/4.6.1b/gcc-12.3.0/openmpi-4.1.6/lib:/lcrc/group/e3sm/soft/improv/netcdf-c/4.9.2b/gcc-12.3.0/openmpi-4.1.6/lib:/opt/pbs/lib:/lcrc/group/e3sm/soft/improv/openmpi/4.1.6/gcc-12.3.0/lib; if [ -z "$LD_LIBRARY_PATH" ]; then echo $lp; else echo "$lp:$LD_LIBRARY_PATH"; fi} - ^lockedfile + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /lcrc/soft/climate/moab/improv/gnu; else echo "$MOAB_ROOT"; fi} + ^lockedfile,individual 128M @@ -2775,9 +2879,9 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss /usr/workspace/e3sm/ccsm3data/inputdata/atm/datm7 /p/lustre2/$USER/archive/$CASE /p/lustre2/$USER/ccsm_baselines/$COMPILER - /usr/workspace/e3sm/tools/cprnc + /usr/workspace/e3sm/apps/cprnc 8 - lc_slurm + slurm boutte3 -at- llnl.gov 56 56 @@ -2786,8 +2890,16 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss srun + + --mpi=pmi2 + --export=ALL + -n {{ total_tasks }} -N {{ num_nodes }} + -c 1 + --cpu_bind=cores + -m plane={{ tasks_per_node }} + - + /usr/share/lmod/lmod/init/env_modules_python.py /usr/share/lmod/lmod/init/perl /usr/share/lmod/lmod/init/sh @@ -2799,24 +2911,27 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss python/3.9.12 git + subversion + cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic - mvapich2/2.3.7 - cmake/3.19.2 - /usr/workspace/e3sm/install/quartz/modulefiles - hdf5/1.12.2 - netcdf-c/4.9.0 - netcdf-fortran/4.6.0 - parallel-netcdf/1.12.3 - screamML-venv/0.0.1 - subversion + /usr/workspace/e3sm/spack/modules/ruby/linux-rhel8-x86_64/Core + mvapich2/2.3.7-ll7cmqm + hdf5/1.10.7-ewjpbjd + netcdf-c/4.4.1.1-vaxofek + netcdf-fortran/4.4.4-3pzbx2u + parallel-netcdf/1.11.0-tzgdala $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld - /usr/workspace/e3sm/install/quartz/netcdf-fortran/ - /usr/tce/packages/parallel-netcdf/parallel-netcdf-1.12.3-mvapich2-2.3.7-intel-classic-2021.6.0 + 128M + FALSE + /usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/hdf5-1.10.7-ewjpbjdhjgjzrzjcvwyjyuulaesbsjhg + /usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/netcdf-c-4.4.1.1-vaxofekwvnvngh7wptmzkwdb7tkzvesn + /usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/netcdf-fortran-4.4.4-3pzbx2unddhladhubaahhhysjmprzqi2 + /usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/parallel-netcdf-1.11.0-tzgdalakmem7tod6cruhqyeackeix5q5 @@ -2831,9 +2946,9 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss /usr/workspace/e3sm/ccsm3data/inputdata/atm/datm7 /p/lustre2/$USER/archive/$CASE /p/lustre2/$USER/ccsm_baselines/$COMPILER - /usr/workspace/e3sm/tools/cprnc + /usr/workspace/e3sm/apps/cprnc 8 - lc_slurm + slurm boutte3 -at- llnl.gov 224 112 @@ -2842,8 +2957,16 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss srun + + --mpi=pmi2 + --export=ALL + -n {{ total_tasks }} -N {{ num_nodes }} + -c 1 + --cpu_bind=cores + -m plane={{ tasks_per_node }} + - + /usr/share/lmod/lmod/init/env_modules_python.py /usr/share/lmod/lmod/init/perl /usr/share/lmod/lmod/init/sh @@ -2855,24 +2978,27 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss python/3.9.12 git + subversion mkl/2022.1.0 intel-classic/2021.6.0-magic - mvapich2/2.3.7 cmake/3.19.2 - /usr/workspace/e3sm/install/quartz/modulefiles - hdf5/1.12.2 - netcdf-c/4.9.0 - netcdf-fortran/4.6.0 - parallel-netcdf/1.12.3 - screamML-venv/0.0.1 - subversion + /usr/workspace/e3sm/spack/modules/dane/linux-rhel8-x86_64/Core + mvapich2/2.3.7-27jao34 + hdf5/1.10.7-766kapa + netcdf-c/4.4.1.1-2uznnlw + netcdf-fortran/4.4.4-itpstyo + parallel-netcdf/1.11.0-26sxm4m $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld - /usr/workspace/e3sm/install/quartz/netcdf-fortran/ - /usr/tce/packages/parallel-netcdf/parallel-netcdf-1.12.3-mvapich2-2.3.7-intel-classic-2021.6.0 + 128M + FALSE + /usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/hdf5-1.10.7-766kapalbrdntu2pcgdgbhg2ch26gsuv + /usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/netcdf-c-4.4.1.1-2uznnlwgiezxute6iyqzqjrpolokeaib + /usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/netcdf-fortran-4.4.4-itpstyordbern7vlulmlnt47eeeokzfp + /usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/parallel-netcdf-1.11.0-26sxm4mormsglmhi24poix7sugbigkck @@ -4387,7 +4513,7 @@ unset MPIR_CVAR_CH4_POSIX_COLL_SELECTION_TUNING_JSON_FILE /usr/projects/e3sm/inputdata/atm/datm7 /lustre/scratch5/$ENV{USER}/E3SM/archive/$CASE /lustre/scratch5/$ENV{USER}/E3SM/input_data/ccsm_baselines/$COMPILER - /usr/projects/climate/SHARED_CLIMATE/software/badger/cprnc + /usr/projects/e3sm/software/chicoma-cpu/cprnc 10 e3sm_developer 4 @@ -4407,11 +4533,11 @@ unset MPIR_CVAR_CH4_POSIX_COLL_SELECTION_TUNING_JSON_FILE - /usr/share/lmod/8.3.1/init/perl + /usr/share/lmod/lmod/init/perl - /usr/share/lmod/8.3.1/init/python - /usr/share/lmod/8.3.1/init/sh - /usr/share/lmod/8.3.1/init/csh + /usr/share/lmod/lmod/init/python + /usr/share/lmod/lmod/init/sh + /usr/share/lmod/lmod/init/csh /usr/share/lmod/lmod/libexec/lmod perl /usr/share/lmod/lmod/libexec/lmod python module @@ -4423,39 +4549,42 @@ unset MPIR_CVAR_CH4_POSIX_COLL_SELECTION_TUNING_JSON_FILE cray-parallel-netcdf cray-netcdf cray-hdf5 - PrgEnv-gnu - PrgEnv-intel - PrgEnv-nvidia - PrgEnv-cray - PrgEnv-aocc + gcc + gcc-native intel intel-oneapi nvidia aocc cudatoolkit climate-utils + cray-libsci craype-accel-nvidia80 craype-accel-host perftools-base perftools darshan + PrgEnv-gnu + PrgEnv-intel + PrgEnv-nvidia + PrgEnv-cray + PrgEnv-aocc - PrgEnv-gnu/8.4.0 - gcc/12.2.0 - cray-libsci/23.05.1.4 + PrgEnv-gnu/8.5.0 + gcc-native/12.3 + cray-libsci/23.12.5 - PrgEnv-nvidia/8.4.0 - nvidia/22.7 - cray-libsci/23.05.1.4 + PrgEnv-nvidia/8.5.0 + nvidia/24.7 + cray-libsci/23.12.5 - PrgEnv-intel/8.4.0 - intel-classic/2023.2.0 + PrgEnv-intel/8.5.0 + intel/2023.2.0 @@ -4466,13 +4595,12 @@ unset MPIR_CVAR_CH4_POSIX_COLL_SELECTION_TUNING_JSON_FILE craype-accel-host - craype/2.7.21 - cray-mpich/8.1.26 - libfabric/1.15.2.0 - cray-hdf5-parallel/1.12.2.3 - cray-netcdf-hdf5parallel/4.9.0.3 - cray-parallel-netcdf/1.12.3.3 - cmake/3.25.1 + craype/2.7.30 + cray-mpich/8.1.28 + cray-hdf5-parallel/1.12.2.9 + cray-netcdf-hdf5parallel/4.9.0.9 + cray-parallel-netcdf/1.12.3.9 + cmake/3.27.7 @@ -4494,14 +4622,16 @@ unset MPIR_CVAR_CH4_POSIX_COLL_SELECTION_TUNING_JSON_FILE $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} + + /usr/lib64/gcc/x86_64-suse-linux/12:$ENV{LD_LIBRARY_PATH} + -1 - Chicoma GPU nodes at LANL IC. Each GPU node has single -AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' + Chicoma GPU nodes at LANL IC. Each GPU node has single AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' ch-fe* Linux gnugpu,gnu,nvidiagpu,nvidia @@ -4511,7 +4641,7 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' /usr/projects/e3sm/inputdata/atm/datm7 /lustre/scratch5/$ENV{USER}/E3SM/archive/$CASE /lustre/scratch5/$ENV{USER}/E3SM/input_data/ccsm_baselines/$COMPILER - /usr/projects/climate/SHARED_CLIMATE/software/badger/cprnc + /usr/projects/e3sm/software/chicoma-cpu/cprnc 10 e3sm_developer 4 @@ -4535,11 +4665,11 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' - /usr/share/lmod/8.3.1/init/perl + /usr/share/lmod/lmod/init/perl - /usr/share/lmod/8.3.1/init/python - /usr/share/lmod/8.3.1/init/sh - /usr/share/lmod/8.3.1/init/csh + /usr/share/lmod/lmod/init/python + /usr/share/lmod/lmod/init/sh + /usr/share/lmod/lmod/init/csh /usr/share/lmod/lmod/libexec/lmod perl /usr/share/lmod/lmod/libexec/lmod python module @@ -4551,32 +4681,35 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' cray-parallel-netcdf cray-netcdf cray-hdf5 - PrgEnv-gnu - PrgEnv-intel - PrgEnv-nvidia - PrgEnv-cray - PrgEnv-aocc intel intel-oneapi nvidia aocc cudatoolkit climate-utils + cray-libsci craype-accel-nvidia80 craype-accel-host perftools-base perftools darshan + PrgEnv-gnu + PrgEnv-intel + PrgEnv-nvidia + PrgEnv-cray + PrgEnv-aocc - PrgEnv-gnu/8.4.0 - gcc/11.2.0 + PrgEnv-gnu/8.5.0 + gcc/12.2.0 + cray-libsci/23.05.1.4 PrgEnv-nvidia/8.4.0 nvidia/22.7 + cray-libsci/23.05.1.4 @@ -4599,14 +4732,13 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' - cray-libsci/23.05.1.4 + craype-accel-host craype/2.7.21 cray-mpich/8.1.26 - libfabric/1.15.2.0 cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 - cmake/3.25.1 + cmake/3.27.7 @@ -4629,6 +4761,9 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' $ENV{CRAY_PARALLEL_NETCDF_PREFIX} /usr/projects/e3sm/cudatoolkit:$ENV{PKG_CONFIG_PATH} + + /opt/cray/pe/gcc/12.2.0/snos/lib64:$ENV{LD_LIBRARY_PATH} + -1 diff --git a/cime_config/testmods_dirs/config_pes_tests.xml b/cime_config/testmods_dirs/config_pes_tests.xml index 1ac88ac5fe6f..eafc8e2a1fe2 100644 --- a/cime_config/testmods_dirs/config_pes_tests.xml +++ b/cime_config/testmods_dirs/config_pes_tests.xml @@ -161,31 +161,41 @@ - tests+anvil: --compset WCYCL* --res ne30pg2_IcoswISC30E3r5 on 16 nodes pure-MPI + tests+anvil: --compset WCYCL* --res ne30pg2_IcoswISC30E3r5 on 25 nodes pure-MPI - 396 - 396 - 396 - 396 - 180 - 396 + 675 + 324 + 324 + 360 + 216 + 684 - 396 + 0 + 360 + 360 + 0 + 684 + 0 tests+anvil: --compset BGC* --res ne30pg2_r05_IcoswISC30E3r5 on 30 nodes pure-MPI 675 - 684 - 684 - 684 - 396 + 324 + 324 + 360 + 216 684 + 0 + 360 + 360 + 0 684 + 0 @@ -203,6 +213,17 @@ -6 + + "tests+anvil, F compset, 6 nodes" + + -16 + -16 + -16 + -16 + -16 + -16 + + @@ -241,17 +262,17 @@ - tests+anvil: --compset WCYCL1850 --res northamericax4v1pg2_WC14to60E2r3 on 64 nodes pure-MPI, 2.133 sypd + tests+anvil: --compset WCYCL1850 --res northamericax4v1pg2_WC14to60E2r3 on 69 nodes pure-MPI, 2.046 sypd - 1800 - 1800 - 1800 - 1800 + 1980 + 1980 + 1980 + 1944 504 - 1800 + 1980 - 1800 + 1980 diff --git a/cime_config/testmods_dirs/io/force_adiosc/shell_commands b/cime_config/testmods_dirs/io/force_adiosc/shell_commands new file mode 100644 index 000000000000..543d5dbfef07 --- /dev/null +++ b/cime_config/testmods_dirs/io/force_adiosc/shell_commands @@ -0,0 +1,2 @@ +#!/bin/bash +./xmlchange PIO_TYPENAME="adiosc" diff --git a/cime_config/tests.py b/cime_config/tests.py index bce0c4fcc265..d67f15e7469b 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -49,6 +49,9 @@ "ERS.f19_g16.I20TRGSWCNPECACNTBC.elm-eca_f19_g16_I20TRGSWCNPECACNTBC", "ERS.f19_g16.I20TRGSWCNPRDCTCBC.elm-ctc_f19_g16_I20TRGSWCNPRDCTCBC", "ERS.r05_r05.ICNPRDCTCBC.elm-cbudget", + "ERS.ELM_USRDAT.I1850CNPRDCTCBC.elm-snowveg_arctic", + "ERS.ELM_USRDAT.I1850CNPRDCTCBC.elm-usrpft_default_I1850CNPRDCTCBC", + "ERS.ELM_USRDAT.I1850CNPRDCTCBC.elm-usrpft_codetest_I1850CNPRDCTCBC", ) }, @@ -94,8 +97,6 @@ "SMS.r05_r05.IELM.elm-topounit", "ERS.ELM_USRDAT.I1850ELM.elm-usrdat", "ERS.r05_r05.IELM.elm-lnd_rof_2way", - "ERS.ELM_USRDAT.I1850CNPRDCTCBC.elm-usrpft_default_I1850CNPRDCTCBC", - "ERS.ELM_USRDAT.I1850CNPRDCTCBC.elm-usrpft_codetest_I1850CNPRDCTCBC", "ERS.r05_r05.IELM.elm-V2_ELM_MOSART_features", "ERS.ELM_USRDAT.IELM.elm-surface_water_dynamics" ) @@ -176,7 +177,7 @@ ) }, - "e3sm_p3_developer" : { + "e3sm_p3_developer" : { "tests" : ( "ERP.ne4pg2_oQU480.F2010.eam-p3", "REP_Ln5.ne4pg2_oQU480.F2010.eam-p3", @@ -188,6 +189,17 @@ "ERS.ne4pg2_oQU480.F2010.eam-p3" ) }, + + "e3sm_orodrag_developer" : { + "tests" : ( + "ERP.ne4pg2_oQU480.F2010.eam-orodrag_ne4pg2", + "REP_Ln5.ne4pg2_oQU480.F2010.eam-orodrag_ne4pg2", + "PET.ne4pg2_oQU480.F2010.eam-orodrag_ne4pg2", + "PEM_Ln18.ne4pg2_oQU480.F2010.eam-orodrag_ne4pg2", + "SMS_Ln5.ne30pg2_EC30to60E2r2.F2010.eam-orodrag_ne30pg2", + "SMS_D_Ln5.ne4pg2_oQU480.F2010.eam-orodrag_ne4pg2" + ) + }, "e3sm_atm_integration" : { "inherit" : ("eam_preqx", "eam_theta"), @@ -207,6 +219,7 @@ "REP_Ln5.ne4pg2_oQU480.F2010", "SMS_Ld3.ne4pg2_oQU480.F2010.eam-thetahy_sl_pg2_mass", "ERP_Ld3.ne4pg2_ne4pg2.FIDEAL.allactive-pioroot1", + "ERS_Ld5.ne4pg2_oQU480.F2010.eam-sathist_F2010", ) }, @@ -266,6 +279,9 @@ "SMS_D_Ld1.T62_oQU240wLI.GMPAS-IAF-PISMF.mpaso-impl_top_drag", "SMS_D_Ld1.T62_oQU240.GMPAS-IAF.mpaso-harmonic_mean_drag", "SMS_D_Ld1.T62_oQU240.GMPAS-IAF.mpaso-upwind_advection", + "ERS_Ld5_D.T62_oQU240.GMPAS-IAF.mpaso-conservation_check", + "ERS_Ld5_PS.ne30pg2_r05_IcoswISC30E3r5.CRYO1850-DISMF.mpaso-scaled_dib_dismf", + "ERS_Ld5.TL319_oQU240wLI_gis20.MPAS_LISIO_JRA1p5.mpaso-ocn_glc_tf_coupling", ) }, @@ -275,7 +291,7 @@ "ERS_P480_Ld5.TL319_IcoswISC30E3r5.GMPAS-JRA1p5-DIB-PISMF.mpaso-jra_1958", "PEM_P480_Ld5.TL319_IcoswISC30E3r5.GMPAS-JRA1p5-DIB-PISMF.mpaso-jra_1958", "SMS_P480_Ld5.TL319_IcoswISC30E3r5.GMPAS-JRA1p5-DIB-PISMF-TMIX.mpaso-jra_1958", - "SMS_P480_Ld5.TL319_IcoswISC30E3r5.GMPAS-JRA1p5-DIB-PISMF-DSGR.mpaso-jra_1958", + "PET_P480_Ld2.TL319_IcoswISC30E3r5.GMPAS-JRA1p5-DIB-PISMF-DSGR.mpaso-jra_1958", ) }, @@ -307,6 +323,8 @@ "ERS_Ld5.TL319_oQU240wLI_ais8to30.MPAS_LISIO_JRA1p5.mpaso-ocn_glcshelf", "SMS_P12x2.ne4pg2_oQU480.WCYCL1850NS.allactive-mach_mods", "ERS_Ln9.ne4pg2_ne4pg2.F2010-MMF1.eam-mmf_crmout", + "SMS_Lh4.ne4_ne4.F2010-SCREAMv1.eamxx-output-preset-1", + "SMS_Lh4.ne4pg2_ne4pg2.F2010-SCREAMv1.eamxx-output-preset-1--eamxx-prod", ) }, @@ -517,6 +535,7 @@ "SMS.ne4pg2_oQU480.F2010.eam-thetanh_ftype2", "SMS.ne4pg2_oQU480.F2010.eam-thetanh_ftype4", "SMS.ne4pg2_oQU480.F2010.eam-thetahy_sl", + "ERS.ne4pg2_oQU480.F2010.eam-thetahy_sl_nsubstep2", "ERS.ne4pg2_oQU480.F2010.eam-thetahy_ftype2", "ERS.ne4pg2_oQU480.F2010.eam-thetanh_ftype2", ) @@ -636,17 +655,17 @@ "time" : "03:00:00", }, - "e3sm_scream" : { + "e3sm_eamxx" : { "time" : "03:00:00", - "inherit" : ("e3sm_scream_v0"), + "inherit" : ("e3sm_eamxx_v0"), }, - "e3sm_scream_v0" : { + "e3sm_eamxx_v0" : { "time" : "03:00:00", - "inherit" : ("e3sm_scream_v0_lowres"), + "inherit" : ("e3sm_eamxx_v0_lowres"), }, - "e3sm_scream_v0_lowres" : { + "e3sm_eamxx_v0_lowres" : { "time" : "03:00:00", "tests" : ( "SMS_D.ne4pg2_ne4pg2.F2010-SCREAM-HR", @@ -657,66 +676,70 @@ ) }, - "e3sm_scream_v1" : { + "e3sm_eamxx_v1" : { "time" : "03:00:00", - "inherit" : ("e3sm_scream_v1_lowres", "e3sm_scream_v1_medres", "e3sm_scream_v1_mpassi"), + "inherit" : ("e3sm_eamxx_v1_lowres", "e3sm_eamxx_v1_medres", "e3sm_eamxx_v1_mpassi"), }, - "e3sm_scream_v1_lowres" : { + "e3sm_eamxx_v1_lowres" : { "time" : "01:00:00", - "inherit" : ("e3sm_scream_mam4xx_v1_lowres"), + "inherit" : ("e3sm_eamxx_mam4xx_v1_lowres"), "tests" : ( - "ERP_D_Lh4.ne4_ne4.F2010-SCREAMv1.scream-output-preset-1", - "ERS_Ln9.ne4_ne4.F2000-SCREAMv1-AQP1.scream-output-preset-2", - "SMS_D_Ln9.ne4_ne4.F2010-SCREAMv1-noAero.scream-output-preset-3", - "ERP_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-output-preset-4", - "ERS_D_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-rad_frequency_2--scream-output-preset-5", - "ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-small_kernels--scream-output-preset-5", - "ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-small_kernels_p3--scream-output-preset-5", - "ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-small_kernels_shoc--scream-output-preset-5", + "ERP_D_Lh4.ne4_ne4.F2010-SCREAMv1.eamxx-output-preset-1", + "ERS_Ln9.ne4_ne4.F2000-SCREAMv1-AQP1.eamxx-output-preset-2", + "SMS_D_Ln9.ne4_ne4.F2010-SCREAMv1-noAero.eamxx-output-preset-3", + "ERP_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.eamxx-output-preset-4", + "ERS_D_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.eamxx-rad_frequency_2--eamxx-output-preset-5", + "ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.eamxx-small_kernels--eamxx-output-preset-5", + "ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.eamxx-small_kernels_p3--eamxx-output-preset-5", + "ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.eamxx-small_kernels_shoc--eamxx-output-preset-5", + "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.eamxx-mam4xx-all_mam4xx_procs", ) }, - "e3sm_scream_v1_dp-eamxx" : { + "e3sm_eamxx_v1_dp-eamxx" : { "time" : "01:00:00", # each test runs with 225 dynamics and 100 physics columns, roughly size of ne2 "tests" : ( - "ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-dycomsrf01", - "ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-arm97", - "ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-comble", + "ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.eamxx-dpxx-dycomsrf01", + "ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.eamxx-dpxx-arm97", + "ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.eamxx-dpxx-comble", "ERS_P16_Ln22.ne30pg2_ne30pg2.FRCE-SCREAMv1-DP", ) }, - # Tests run on exclusively on mappy for scream AT testing. These tests + # Tests run on exclusively on mappy for eamxx AT testing. These tests # should be fast, so we limit it to low res and add some thread tests # specifically for mappy. - "e3sm_scream_v1_at" : { - "inherit" : ("e3sm_scream_v1_lowres", "e3sm_scream_v1_dp-eamxx"), - "tests" : ("PET_Ln9_P32x2.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-output-preset-1") + "e3sm_eamxx_v1_at" : { + "inherit" : ("e3sm_eamxx_v1_lowres", "e3sm_eamxx_v1_dp-eamxx"), + "tests" : ("PET_Ln9_P32x2.ne4pg2_ne4pg2.F2010-SCREAMv1.eamxx-output-preset-1") }, - "e3sm_scream_v1_medres" : { + "e3sm_eamxx_v1_medres" : { "time" : "02:00:00", "tests" : ( # "SMS_D_Ln2.ne30_ne30.F2000-SCREAMv1-AQP1", # Uncomment once IC file for ne30 is ready - "ERS_Ln22.ne30_ne30.F2010-SCREAMv1.scream-internal_diagnostics_level--scream-output-preset-3", - "PEM_Ln90.ne30pg2_ne30pg2.F2010-SCREAMv1.scream-spa_remap--scream-output-preset-4", - "ERS_Ln90.ne30pg2_ne30pg2.F2010-SCREAMv1.scream-small_kernels--scream-output-preset-5", - "ERP_Ln22.conusx4v1pg2_r05_oECv3.F2010-SCREAMv1-noAero.scream-bfbhash--scream-output-preset-6", + "ERS_Ln22.ne30_ne30.F2010-SCREAMv1.eamxx-internal_diagnostics_level--eamxx-output-preset-3", + "PEM_Ln90.ne30pg2_ne30pg2.F2010-SCREAMv1.eamxx-spa_remap--eamxx-output-preset-4", + "ERS_Ln90.ne30pg2_ne30pg2.F2010-SCREAMv1.eamxx-small_kernels--eamxx-output-preset-5", + "ERP_Ln22.conusx4v1pg2_r05_oECv3.F2010-SCREAMv1-noAero.eamxx-bfbhash--eamxx-output-preset-6", + "ERS_Ln22.ne30pg2_ne30pg2.F2010-SCREAMv1.eamxx-L128--eamxx-output-preset-4", + "REP_Ld5.ne30pg2_ne30pg2.F2010-SCREAMv1.eamxx-L128--eamxx-output-preset-6", + "ERS_Ln90.ne30pg2_ne30pg2.F2010-SCREAMv1.eamxx-L128--eamxx-sl_nsubstep2", ) }, # Used to track performance - "e3sm_scream_v1_hires" : { + "e3sm_eamxx_v1_hires" : { "time" : "01:00:00", "tests" : ( - "SMS_Ln300.ne30pg2_ne30pg2.F2010-SCREAMv1.scream-perf_test--scream-output-preset-1" + "SMS_Ln300.ne30pg2_ne30pg2.F2010-SCREAMv1.eamxx-perf_test--eamxx-output-preset-1" ) }, - "e3sm_scream_v1_mpassi" : { + "e3sm_eamxx_v1_mpassi" : { "time" : "01:00:00", "tests" : ( # "ERP_D_Ln9.ne4_oQU240.F2010-SCREAMv1-MPASSI.atmlndactive-rtm_off", @@ -724,14 +747,14 @@ # Disable the two 111422-commented tests b/c they fail on pm-gpu and # we're not using MPASSI right now. #111422 "ERP_Ln22.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.atmlndactive-rtm_off", - "ERS_D_Ln22.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.atmlndactive-rtm_off--scream-output-preset-1", + "ERS_D_Ln22.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.atmlndactive-rtm_off--eamxx-output-preset-1", # "ERS_Ln22.ne30_oECv3.F2010-SCREAMv1-MPASSI.atmlndactive-rtm_off", #111422 "PEM_Ln90.ne30pg2_EC30to60E2r2.F2010-SCREAMv1-MPASSI", # "ERS_Ln22.ne30pg2_EC30to60E2r2.F2010-SCREAMv1-MPASSI.atmlndactive-rtm_off", ) }, - "e3sm_scream_v1_long" : { + "e3sm_eamxx_v1_long" : { "time" : "01:00:00", "tests" : ( "ERP_D_Lh182.ne4pg2_ne4pg2.F2010-SCREAMv1", @@ -739,7 +762,7 @@ ) }, - "e3sm_scream_v1_long_crusher" : { + "e3sm_eamxx_v1_long_crusher" : { # _D builds take a long longer on crusher than ascent or pm-gpu, so # don't run the long _D test. "time" : "01:00:00", @@ -748,17 +771,32 @@ ) }, - "e3sm_scream_mam4xx_v1_lowres" : { + "e3sm_eamxx_mam4xx_v1_lowres" : { "time" : "01:00:00", "tests" : ( - "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-optics", - "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-aci", - "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-wetscav", - "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-drydep", + "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.eamxx-mam4xx-optics", + "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.eamxx-mam4xx-aci", + "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.eamxx-mam4xx-wetscav", + "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.eamxx-mam4xx-drydep", + "SMS_D_Ln5.ne30pg2_oECv3.F2010-SCREAMv1-MPASSI.eamxx-mam4xx-remap_emiss_ne4_ne30" + ) + }, + + "e3sm_moab_dev" : { + "time" : "01:00:00", + "tests" : ( + "ERS_Vmoab_Ld3.ne4pg2_r05_oQU480.WCYCL1850NS", + "ERS_Vmoab_Ld3.ne4pg2_oQU480.WCYCL1850NS", + "ERS_Vmoab_Ld3.ne4pg2_oQU480.F1850", + "ERS_Vmoab_Ld3.ne4pg2_ne4pg2.I1850CNPRDCTCBCTOP", + "ERS_Vmoab_Ld3.T62_oQU240wLI.GMPAS-IAF", + "ERS_Vmoab_Ld3.T62_oQU120.CMPASO-NYF", + "ERS_Vmoab_Ld3.r05_r05.RMOSGPCC", ) }, + "e3sm_gpuacc" : { "tests" : ( "SMS_Ld1.T62_oEC60to30v3.CMPASO-NYF", @@ -817,9 +855,13 @@ }, "e3sm_wav_developer" : { - "time" : "0:45:00", + "time" : "1:00:00", "tests" : ( - "ERS.T62_oEC60to30v3_wQU225EC60to30.GMPAS-IAF-WW3", + "SMS_D_Ln3.TL319_EC30to60E2r2_wQU225EC30to60E2r2.GMPAS-JRA1p5-WW3.ww3-jra_1958", + "ERS.ne30pg2_IcoswISC30E3r5_wQU225Icos30E3r5.WCYCL1850-WW3", + "PEM_P480.ne30pg2_IcoswISC30E3r5_wQU225Icos30E3r5.WCYCL1850-WW3", + "PET.ne30pg2_IcoswISC30E3r5_wQU225Icos30E3r5.WCYCL1850-WW3", + "SMS_D_Ln3.ne30pg2_IcoswISC30E3r5_wQU225Icos30E3r5.WCYCL1850-WW3", ) }, diff --git a/components/cmake/modules/FindPIO.cmake b/components/cmake/modules/FindPIO.cmake index 0277918ac8e0..5589dff0ed04 100644 --- a/components/cmake/modules/FindPIO.cmake +++ b/components/cmake/modules/FindPIO.cmake @@ -42,6 +42,9 @@ endif() # we can assume that an MPI case with ADIOS2_ROOT set is probably # using adios. if (NOT MPILIB STREQUAL "mpi-serial" AND DEFINED ENV{ADIOS2_ROOT}) + if(DEFINED ENV{BLOSC2_ROOT}) + set(ENV{Blosc2_DIR} "$ENV{BLOSC2_ROOT}") + endif() find_package(MPI REQUIRED COMPONENTS C) find_package(ADIOS2 REQUIRED COMPONENTS C) list(APPEND PIOLIBS adios2::adios2) diff --git a/components/data_comps/datm/cime_config/config_component.xml b/components/data_comps/datm/cime_config/config_component.xml index 9145bdf4b16d..5b884ecd1ad1 100644 --- a/components/data_comps/datm/cime_config/config_component.xml +++ b/components/data_comps/datm/cime_config/config_component.xml @@ -10,7 +10,7 @@ This file may have atm desc entries. --> - Data driven ATM + Data driven ATM QIAN data set QIAN with water isotopes CRUNCEP data set @@ -28,8 +28,9 @@ COREv2 normal year forcing COREv2 interannual forcing interannual JRA55 forcing - interannual JRA55 forcing, v1.5, through 2020 interannual JRA55 forcing, v1.4, through 2018 + interannual JRA55 forcing, v1.5, through 2020 + interannual JRA55 forcing, v1.5, through 2023 JRA55 Repeat Year Forcing v1.3 1984-1985 JRA55 Repeat Year Forcing v1.3 1990-1991 JRA55 Repeat Year Forcing v1.3 2003-2004 @@ -63,8 +64,9 @@ data (see cime issue #3653 -- https://github.com/ESMCI/cime/issues/3653). CORE2_NYF CORE2_IAF CORE_IAF_JRA - IAF_JRA_1p5 CORE_IAF_JRA_1p4_2018 + IAF_JRA_1p5 + IAF_JRA_1p5 CORE_RYF8485_JRA CORE_RYF9091_JRA CORE_RYF0304_JRA @@ -391,6 +393,7 @@ data (see cime issue #3653 -- https://github.com/ESMCI/cime/issues/3653). $DATM_CLMNCEP_YR_START 1 1 + 1 $DATM_CLMNCEP_YR_START $DATM_CLMNCEP_YR_START $DATM_CLMNCEP_YR_START @@ -488,6 +491,7 @@ data (see cime issue #3653 -- https://github.com/ESMCI/cime/issues/3653). 2003 2016 2020 + 2023 1979 1979 diff --git a/components/data_comps/dice/src/ice_comp_mct.F90 b/components/data_comps/dice/src/ice_comp_mct.F90 index f8c48c89240a..3cd5557f1f23 100644 --- a/components/data_comps/dice/src/ice_comp_mct.F90 +++ b/components/data_comps/dice/src/ice_comp_mct.F90 @@ -79,16 +79,6 @@ subroutine ice_init_mct( EClock, cdata, x2i, i2x, NLFilename ) logical :: scmMode = .false. ! single column mode real(R8) :: scmLat = shr_const_SPVAL ! single column lat real(R8) :: scmLon = shr_const_SPVAL ! single column lon -#ifdef HAVE_MOAB - character(CL) :: filePath ! generic file path - character(CL) :: fileName ! generic file name - character(CS) :: timeName ! domain file: time variable name - character(CS) :: lonName ! domain file: lon variable name - character(CS) :: latName ! domain file: lat variable name - character(CS) :: hgtName ! domain file: hgt variable name - character(CS) :: maskName ! domain file: mask variable name - character(CS) :: areaName ! domain file: area variable name -#endif character(*), parameter :: subName = "(ice_init_mct) " !------------------------------------------------------------------------------- @@ -171,12 +161,9 @@ subroutine ice_init_mct( EClock, cdata, x2i, i2x, NLFilename ) #ifdef HAVE_MOAB if (my_task == master_task) then - call shr_stream_getDomainInfo(SDICE%stream(1), filePath,fileName,timeName,lonName, & - latName,hgtName,maskName,areaName) - call shr_stream_getFile(filePath,fileName) ! send path of ice domain to MOAB coupler. - call seq_infodata_PutData( infodata, ice_domain=fileName) - write(logunit,*), ' filename: ', filename + write(logunit,*), ' file used for ice domain ', SDICE%domainFile + call seq_infodata_PutData( infodata, ice_domain=SDICE%domainFile) endif #endif !---------------------------------------------------------------------------- diff --git a/components/data_comps/dlnd/src/dlnd_comp_mod.F90 b/components/data_comps/dlnd/src/dlnd_comp_mod.F90 index bf723259be48..fbd6e35cf87d 100644 --- a/components/data_comps/dlnd/src/dlnd_comp_mod.F90 +++ b/components/data_comps/dlnd/src/dlnd_comp_mod.F90 @@ -29,6 +29,10 @@ module dlnd_comp_mod use dlnd_shr_mod , only: domain_fracname ! namelist input use dlnd_shr_mod , only: nullstr +#ifdef HAVE_MOAB + use seq_comm_mct, only : mlnid ! id of moab lnd app + use iso_c_binding +#endif ! !PUBLIC TYPES: implicit none private ! except @@ -100,6 +104,15 @@ subroutine dlnd_comp_init(Eclock, x2l, l2x, & scmMode, scmlat, scmlon) ! !DESCRIPTION: initialize dlnd model +#ifdef HAVE_MOAB + use iMOAB, only: iMOAB_DefineTagStorage, & + iMOAB_SetIntTagStorage, iMOAB_SetDoubleTagStorage, & + iMOAB_ResolveSharedEntities, iMOAB_CreateVertices, & + iMOAB_UpdateMeshInfo +#ifdef MOABDEBUG + use iMOAB, only: iMOAB_WriteMesh +#endif +#endif implicit none ! !INPUT/OUTPUT PARAMETERS: @@ -135,6 +148,18 @@ subroutine dlnd_comp_init(Eclock, x2l, l2x, & character(nec_len) :: nec_str ! elevation class, as character string character(*), parameter :: domain_fracname_unset = 'null' +#ifdef HAVE_MOAB + character*400 tagname + real(R8) latv, lonv + integer iv, tagindex, ilat, ilon + real(R8), allocatable, target :: data(:) + integer(IN), pointer :: idata(:) ! temporary + real(R8), dimension(:), allocatable :: moab_vert_coords ! temporary +#ifdef MOABDEBUG + character*100 outfile, wopts +#endif +#endif + !--- formats --- character(*), parameter :: F00 = "('(dlnd_comp_init) ',8a)" character(*), parameter :: F0L = "('(dlnd_comp_init) ',a, l2)" @@ -256,6 +281,119 @@ subroutine dlnd_comp_init(Eclock, x2l, l2x, & call t_stopf('dlnd_initmctdom') +#ifdef HAVE_MOAB + ilat = mct_aVect_indexRA(ggrid%data,'lat') + ilon = mct_aVect_indexRA(ggrid%data,'lon') + allocate(moab_vert_coords(lsize*3)) + do iv = 1, lsize + lonv = ggrid%data%rAttr(ilon, iv) * SHR_CONST_PI/180. + latv = ggrid%data%rAttr(ilat, iv) * SHR_CONST_PI/180. + moab_vert_coords(3*iv-2)=COS(latv)*COS(lonv) + moab_vert_coords(3*iv-1)=COS(latv)*SIN(lonv) + moab_vert_coords(3*iv )=SIN(latv) + enddo + + ! create the vertices with coordinates from MCT domain + ierr = iMOAB_CreateVertices(mlnid, lsize*3, 3, moab_vert_coords) + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to create MOAB vertices in data lnd model') + + tagname='GLOBAL_ID'//C_NULL_CHAR + ierr = iMOAB_DefineTagStorage(mlnid, tagname, & + 0, & ! dense, integer + 1, & ! number of components + tagindex ) + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to retrieve GLOBAL_ID tag ') + + ! get list of global IDs for Dofs + call mct_gsMap_orderedPoints(gsMap, my_task, idata) + + ierr = iMOAB_SetIntTagStorage ( mlnid, tagname, lsize, & + 0, & ! vertex type + idata) + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to set GLOBAL_ID tag ') + + ierr = iMOAB_ResolveSharedEntities( mlnid, lsize, idata ); + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to resolve shared entities') + + deallocate(moab_vert_coords) + deallocate(idata) + + ierr = iMOAB_UpdateMeshInfo( mlnid ) + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to update mesh info ') + + allocate(data(lsize)) + ierr = iMOAB_DefineTagStorage( mlnid, "area:aream:frac:mask"//C_NULL_CHAR, & + 1, & ! dense, double + 1, & ! number of components + tagindex ) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to create tag: area:aream:frac:mask' ) + + data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'area'),:) + tagname='area'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsize, & + 0, & ! set data on vertices + data) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to get area tag ') + + ! set the same data for aream (model area) as area + ! data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'aream'),:) + tagname='aream'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsize, & + 0, & ! set data on vertices + data) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to set aream tag ') + + data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'mask'),:) + tagname='mask'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsize, & + 0, & ! set data on vertices + data) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to set mask tag ') + + data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'frac'),:) + tagname='frac'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsize, & + 0, & ! set data on vertices + data) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to set frac tag ') + + deallocate(data) + + ! define tags + ierr = iMOAB_DefineTagStorage( mlnid, trim(seq_flds_x2l_fields)//C_NULL_CHAR, & + 1, & ! dense, double + 1, & ! number of components + tagindex ) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to create seq_flds_x2l_fields tags ') + + ierr = iMOAB_DefineTagStorage( mlnid, trim(seq_flds_l2x_fields)//C_NULL_CHAR, & + 1, & ! dense, double + 1, & ! number of components + tagindex ) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to create seq_flds_l2x_fields tags ') +#ifdef MOABDEBUG + ! debug test + outfile = 'LndDataMesh.h5m'//C_NULL_CHAR + wopts = ';PARALLEL=WRITE_PART'//C_NULL_CHAR ! + ! write out the mesh file to disk + ierr = iMOAB_WriteMesh(mlnid, trim(outfile), trim(wopts)) + if (ierr .ne. 0) then + call shr_sys_abort(subname//' ERROR in writing data mesh lnd ') + endif +#endif +#endif !---------------------------------------------------------------------------- ! Initialize MCT attribute vectors !---------------------------------------------------------------------------- @@ -339,8 +477,15 @@ subroutine dlnd_comp_run(EClock, x2l, l2x, & inst_suffix, logunit, case_name) ! !DESCRIPTION: run method for dlnd model - implicit none +#ifdef HAVE_MOAB +#ifdef MOABDEBUG + use iMOAB, only: iMOAB_WriteMesh +#endif + use seq_flds_mod , only: seq_flds_l2x_fields + use seq_flds_mod , only: moab_set_tag_from_av +#endif + implicit none ! !INPUT/OUTPUT PARAMETERS: type(ESMF_Clock) , intent(in) :: EClock type(mct_aVect) , intent(inout) :: x2l @@ -366,6 +511,17 @@ subroutine dlnd_comp_run(EClock, x2l, l2x, & integer(IN) :: nu ! unit number logical :: write_restart ! restart now character(len=18) :: date_str +#ifdef HAVE_MOAB + real(R8), allocatable, target :: datam(:) + type(mct_list) :: temp_list + integer :: size_list, index_list, lsize + type(mct_string) :: mctOStr ! + character*400 tagname, mct_field +#ifdef MOABDEBUG + integer :: cur_dlnd_stepno, ierr + character*100 outfile, wopts, lnum +#endif +#endif character(*), parameter :: F00 = "('(dlnd_comp_run) ',8a)" character(*), parameter :: F04 = "('(dlnd_comp_run) ',2a,2i8,'s')" @@ -464,6 +620,32 @@ subroutine dlnd_comp_run(EClock, x2l, l2x, & call t_stopf('DLND_RUN') +#ifdef HAVE_MOAB + lsize = mct_avect_lsize(l2x) ! is it the same as mct_avect_lsize(avstrm) ? + allocate(datam(lsize)) ! + call mct_list_init(temp_list ,seq_flds_l2x_fields) + size_list=mct_list_nitem (temp_list) + do index_list = 1, size_list + call mct_list_get(mctOStr,index_list,temp_list) + mct_field = mct_string_toChar(mctOStr) + tagname= trim(mct_field)//C_NULL_CHAR + call moab_set_tag_from_av(tagname, l2x, index_list, mlnid, datam, lsize) ! loop over all a2x fields, not just a few + enddo + call mct_list_clean(temp_list) + deallocate(datam) ! maybe we should keep it around, deallocate at the final only? + +#ifdef MOABDEBUG + call seq_timemgr_EClockGetData( EClock, stepno=cur_dlnd_stepno ) + write(lnum,"(I0.2)")cur_dlnd_stepno + outfile = 'dlnd_comp_run_'//trim(lnum)//'.h5m'//C_NULL_CHAR + wopts = 'PARALLEL=WRITE_PART'//C_NULL_CHAR + ierr = iMOAB_WriteMesh(mlnid, outfile, wopts) + if (ierr > 0 ) then + write(logunit,*) 'Failed to write data lnd component state ' + endif +#endif +#endif + end subroutine dlnd_comp_run !=============================================================================== diff --git a/components/data_comps/dlnd/src/lnd_comp_mct.F90 b/components/data_comps/dlnd/src/lnd_comp_mct.F90 index f5193ca84580..b699ec217f00 100644 --- a/components/data_comps/dlnd/src/lnd_comp_mct.F90 +++ b/components/data_comps/dlnd/src/lnd_comp_mct.F90 @@ -16,7 +16,11 @@ module lnd_comp_mct use dlnd_comp_mod , only: dlnd_comp_init, dlnd_comp_run, dlnd_comp_final use dlnd_shr_mod , only: dlnd_shr_read_namelists use seq_flds_mod , only: seq_flds_x2l_fields, seq_flds_l2x_fields - +#ifdef HAVE_MOAB + use seq_comm_mct, only : mlnid ! iMOAB app id for lnd + use iso_c_binding + use iMOAB , only: iMOAB_RegisterApplication +#endif ! !PUBLIC TYPES: implicit none private ! except @@ -52,7 +56,9 @@ module lnd_comp_mct !=============================================================================== subroutine lnd_init_mct( EClock, cdata, x2l, l2x, NLFilename ) - +#ifdef HAVE_MOAB + use shr_stream_mod, only: shr_stream_getDomainInfo, shr_stream_getFile +#endif ! !DESCRIPTION: initialize dlnd model implicit none @@ -146,13 +152,25 @@ subroutine lnd_init_mct( EClock, cdata, x2l, l2x, NLFilename ) !---------------------------------------------------------------------------- ! Initialize dlnd !---------------------------------------------------------------------------- - +#ifdef HAVE_MOAB + ierr = iMOAB_RegisterApplication(trim("DLND")//C_NULL_CHAR, mpicom, compid, mlnid) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in registering data lnd comp' + call shr_sys_abort(subname//' ERROR in registering data lnd comp') + endif +#endif call dlnd_comp_init(Eclock, x2l, l2x, & seq_flds_x2l_fields, seq_flds_l2x_fields, & SDLND, gsmap, ggrid, mpicom, compid, my_task, master_task, & inst_suffix, inst_name, logunit, read_restart, & scmMode, scmlat, scmlon) - +#ifdef HAVE_MOAB + if (my_task == master_task) then + call seq_infodata_PutData( infodata, lnd_domain=SDLND%domainFile) ! we use the same one for regular case + ! in regular case, it is copied from fatmlndfrc ; so we don't know if it is data land or not + write(logunit,*), ' use this land domain file: ', SDLND%domainFile + endif +#endif !---------------------------------------------------------------------------- ! Fill infodata that needs to be returned from dlnd !---------------------------------------------------------------------------- diff --git a/components/data_comps/docn/cime_config/config_component.xml b/components/data_comps/docn/cime_config/config_component.xml index 5a30c69df6cf..68ec6a18c9f9 100644 --- a/components/data_comps/docn/cime_config/config_component.xml +++ b/components/data_comps/docn/cime_config/config_component.xml @@ -13,10 +13,11 @@ This file may have ocn desc entries. --> - DOCN + DOCN null mode prescribed ocean mode slab ocean mode + relaxed slab ocean mode aquaplanet slab ocean mode interannual mode aquaplanet mode: @@ -45,12 +46,13 @@ char - prescribed,sst_aquap1,sst_aquap2,sst_aquap3,sst_aquap4,sst_aquap5,sst_aquap6,sst_aquap7,sst_aquap8,sst_aquap9,sst_aquap10,sst_aquapfile,som,som_aquap,sst_aquap_constant,interannual,null + prescribed,sst_aquap1,sst_aquap2,sst_aquap3,sst_aquap4,sst_aquap5,sst_aquap6,sst_aquap7,sst_aquap8,sst_aquap9,sst_aquap10,sst_aquap11,sst_aquap12,sst_aquap13,sst_aquap14,sst_aquap15,sst_aquapfile,som,rso,som_aquap,sst_aquap_constant,interannual,null prescribed null prescribed som + rso som_aquap interannual sst_aquap1 @@ -63,6 +65,12 @@ sst_aquap8 sst_aquap9 sst_aquap10 + + sst_aquap11 + sst_aquap12 + sst_aquap13 + sst_aquap14 + sst_aquap15 sst_aquapfile sst_aquap_constant diff --git a/components/data_comps/docn/cime_config/namelist_definition_docn.xml b/components/data_comps/docn/cime_config/namelist_definition_docn.xml index 948902e37324..fb28b7471c50 100644 --- a/components/data_comps/docn/cime_config/namelist_definition_docn.xml +++ b/components/data_comps/docn/cime_config/namelist_definition_docn.xml @@ -32,6 +32,7 @@ Currently the following datamods are supported prescribed SSTDATA (Run with prescribed SST, ICE_COV) som SOM (Slab ocean model) + rso RSO (Relaxed slab ocean model) null NULL (NULL mode) --> @@ -59,6 +60,7 @@ aquapfile '' som + rso som interannual @@ -93,6 +95,7 @@ null $DIN_LOC_ROOT/ocn/docn7/AQUAPLANET/ $DIN_LOC_ROOT/ocn/docn7/SOM + / $DIN_LOC_ROOT/atm/cam/sst @@ -106,6 +109,7 @@ null $DOCN_AQP_FILENAME $DOCN_SOM_FILENAME + $SSTICE_GRID_FILENAME sst_HadOIBl_bc_1x1_1850_2014_c150416.nc @@ -145,6 +149,7 @@ null $DIN_LOC_ROOT/ocn/docn7/AQUAPLANET $DIN_LOC_ROOT/ocn/docn7/SOM + / $DIN_LOC_ROOT/atm/cam/sst @@ -158,6 +163,7 @@ null $DOCN_AQP_FILENAME $DOCN_SOM_FILENAME + $SSTICE_DATA_FILENAME sst_HadOIBl_bc_1x1_1850_2014_c150416.nc @@ -181,6 +187,10 @@ hblt h qdp qbot + + SST_cpl t + hblt h + SST_cpl t @@ -213,6 +223,7 @@ $SSTICE_YEAR_ALIGN 0 1 + $SSTICE_YEAR_ALIGN 1 @@ -227,6 +238,7 @@ $SSTICE_YEAR_START 0 1 + $SSTICE_YEAR_START 1850 @@ -241,6 +253,7 @@ $SSTICE_YEAR_END 0 1 + $SSTICE_YEAR_END 2014 @@ -257,7 +270,7 @@ char streams shr_strdata_nml - SSTDATA,SST_AQUAP1,SST_AQUAP2,SST_AQUAP3,SST_AQUAP4,SST_AQUAP5,SST_AQUAP6,SST_AQUAP7,SST_AQUAP8,SST_AQUAP9,SST_AQUAP10,SST_AQUAPFILE,SST_AQUAP_CONSTANT,SOM,SOM_AQUAP,IAF,NULL,COPYALL + SSTDATA,SST_AQUAP1,SST_AQUAP2,SST_AQUAP3,SST_AQUAP4,SST_AQUAP5,SST_AQUAP6,SST_AQUAP7,SST_AQUAP8,SST_AQUAP9,SST_AQUAP10,SST_AQUAP11,SST_AQUAP12,SST_AQUAP13,SST_AQUAP14,SST_AQUAP15,SST_AQUAPFILE,SST_AQUAP_CONSTANT,SOM,RSO,SOM_AQUAP,IAF,NULL,COPYALL General method that operates on the data. This is generally implemented in the data models but is set in the strdata method for @@ -323,9 +336,15 @@ SST_AQUAP8 SST_AQUAP9 SST_AQUAP10 + SST_AQUAP11 + SST_AQUAP12 + SST_AQUAP13 + SST_AQUAP14 + SST_AQUAP15 SST_AQUAPFILE SST_AQUAP_CONSTANT SOM + RSO SOM_AQUAP IAF @@ -657,4 +676,29 @@ + + real(30) + docn + docn_nml + + Relaxation timescale for relaxed slab ocean (RSO) mode + + + 691200 + + + + + real(30) + docn + docn_nml + + globally fixed mixed layer depth (MLD) for relaxed slab ocean (RSO) mode + use -1 to disable - input data file should have hblt field to override this + + + 50 + + + diff --git a/components/data_comps/docn/src/docn_comp_mod.F90 b/components/data_comps/docn/src/docn_comp_mod.F90 index e692882c9db1..00a7ef4338c0 100644 --- a/components/data_comps/docn/src/docn_comp_mod.F90 +++ b/components/data_comps/docn/src/docn_comp_mod.F90 @@ -29,6 +29,8 @@ module docn_comp_mod use docn_shr_mod , only: rest_file ! namelist input use docn_shr_mod , only: rest_file_strm ! namelist input use docn_shr_mod , only: sst_constant_value ! namelist input + use docn_shr_mod , only: RSO_relax_tau ! namelist input for relaxed slab ocean (RSO) + use docn_shr_mod , only: RSO_fixed_MLD ! namelist input for relaxed slab ocean (RSO) use docn_shr_mod , only: nullstr #ifdef HAVE_MOAB @@ -70,6 +72,8 @@ module docn_comp_mod integer(IN) :: kt,ks,ku,kv,kdhdx,kdhdy,kq,kswp ! field indices integer(IN) :: kswnet,klwup,klwdn,ksen,klat,kmelth,ksnow,krofi integer(IN) :: kh,kqbot + integer(IN) :: k10uu ! index for u10 + integer(IN) :: kRSO_bckgrd_sst ! index for background SST (relaxed slab ocean) integer(IN) :: index_lat, index_lon integer(IN) :: kmask, kfrac ! frac and mask field indices of docn domain integer(IN) :: ksomask ! So_omask field index @@ -93,7 +97,7 @@ module docn_comp_mod character(12) , parameter :: avofld(1:ktrans) = & (/ "So_t ","So_u ","So_v ","So_dhdx ",& "So_dhdy ","So_s ","strm_h ","strm_qbot "/) - character(len=*), parameter :: flds_strm = 'strm_h:strm_qbot' + character(len=*),parameter :: flds_strm = 'strm_h:strm_qbot:So_t' !-------------------------------------------------------------------------- !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -281,7 +285,7 @@ subroutine docn_comp_init(Eclock, x2o, o2x, & kdhdx = mct_aVect_indexRA(o2x,'So_dhdx') kdhdy = mct_aVect_indexRA(o2x,'So_dhdy') kswp = mct_aVect_indexRA(o2x,'So_fswpen', perrwith='quiet') - kq = mct_aVect_indexRA(o2x,'Fioo_q') + kq = mct_aVect_indexRA(o2x,'Fioo_q') ! ocn freezing melting potential call mct_aVect_init(x2o, rList=seq_flds_x2o_fields, lsize=lsize) call mct_aVect_zero(x2o) @@ -294,12 +298,14 @@ subroutine docn_comp_init(Eclock, x2o, o2x, & klwdn = mct_aVect_indexRA(x2o,'Faxa_lwdn') ksnow = mct_aVect_indexRA(x2o,'Faxa_snow') kmelth = mct_aVect_indexRA(x2o,'Fioi_melth') + k10uu = mct_aVect_indexRA(x2o,'So_duu10n') call mct_aVect_init(avstrm, rList=flds_strm, lsize=lsize) call mct_aVect_zero(avstrm) kh = mct_aVect_indexRA(avstrm,'strm_h') kqbot = mct_aVect_indexRA(avstrm,'strm_qbot') + kRSO_bckgrd_sst = mct_aVect_indexRA(avstrm,'So_t') allocate(somtp(lsize)) allocate(tfreeze(lsize)) @@ -472,14 +478,17 @@ subroutine docn_comp_init(Eclock, x2o, o2x, & call shr_mpi_bcast(exists,mpicom,'exists') call shr_mpi_bcast(exists1,mpicom,'exists1') - if (trim(datamode) == 'SOM' .or. trim(datamode) == 'SOM_AQUAP') then - if (exists1) then - if (my_task == master_task) write(logunit,F00) ' reading ',trim(rest_file) - call shr_pcdf_readwrite('read',SDOCN%pio_subsystem, SDOCN%io_type, & - trim(rest_file), mpicom, gsmap=gsmap, rf1=somtp, rf1n='somtp', io_format=SDOCN%io_format) - else - if (my_task == master_task) write(logunit,F00) ' file not found, skipping ',trim(rest_file) - endif + if ( trim(datamode) == 'SOM' & ! Traditional slab ocean + .or. trim(datamode) == 'RSO' & ! Relaxed slab ocean + .or. trim(datamode) == 'SOM_AQUAP' & ! Aquaplanet slab ocean + ) then + if (exists1) then + if (my_task == master_task) write(logunit,F00) ' reading ',trim(rest_file) + call shr_pcdf_readwrite('read',SDOCN%pio_subsystem, SDOCN%io_type, & + trim(rest_file), mpicom, gsmap=gsmap, rf1=somtp, rf1n='somtp', io_format=SDOCN%io_format) + else + if (my_task == master_task) write(logunit,F00) ' file not found, skipping ',trim(rest_file) + endif endif if (exists) then @@ -562,7 +571,18 @@ subroutine docn_comp_run(EClock, x2o, o2x, & integer(IN) :: idt ! integer timestep real(R8) :: dt ! timestep integer(IN) :: nu ! unit number - real(R8) :: hn ! h field + real(R8) :: hn ! h field - mixed layer depth (MLD) + ! relaxed slab ocean mode variables + real(R8) :: RSO_bckgrd_sst ! background SST + real(R8) :: RSO_X_cool ! logistics function weight + real(R8) :: u10 ! 10 m wind + ! relaxed slab ocean fixed parameters + integer, parameter :: RSO_slab_option = 0 ! Option for setting RSO_X_cool + real(R8), parameter :: RSO_R_cool = 11.75_r8/86400._r8 ! base cooling rate [K/s] + real(R8), parameter :: RSO_Tdeep = 271.0_r8 ! deep water temperature [K] + real(R8), parameter :: RSO_dT_o = 27.0_r8 ! scaling temperature gradient + real(R8), parameter :: RSO_h_o = 30.0_r8 ! scaling mixed layer depth + character(len=18) :: date_str character(len=CL) :: local_case_name real(R8), parameter :: & @@ -768,6 +788,65 @@ subroutine docn_comp_run(EClock, x2o, o2x, & enddo endif ! firstcall + ! Relaxed Slab Ocean based on Zarzycki(2016) + ! Zarzycki, C. M., 2016: Tropical Cyclone Intensity Errors Associated with Lack of Two-Way Ocean Coupling in High-Resolution Global Simulations. J. Climate, 29, 8589–8610. + ! https://journals.ametsoc.org/view/journals/clim/29/23/jcli-d-16-0273.1.xml + case('RSO') + lsize = mct_avect_lsize(o2x) + do n = 1,SDOCN%nstreams + call shr_dmodel_translateAV(SDOCN%avs(n),avstrm,avifld,avofld,rearr) + enddo + if (firstcall) then + do n = 1,lsize + if (.not. read_restart) then + somtp(n) = o2x%rAttr(kt,n) + TkFrz + endif + o2x%rAttr(kt,n) = somtp(n) + o2x%rAttr(kq,n) = 0.0_R8 + enddo + else ! firstcall + tfreeze = shr_frz_freezetemp(o2x%rAttr(ks,:)) + TkFrz + do n = 1,lsize + if (imask(n) /= 0) then + !******************************************************************* + if (RSO_fixed_MLD>=0) then + hn = RSO_fixed_MLD + else + hn = avstrm%rAttr(kh,n) + endif + ! Get "background" temperature for relaxation + RSO_bckgrd_sst = avstrm%rAttr(kRSO_bckgrd_sst,n) + TkFrz + u10 = SQRT(x2o%rAttr(k10uu,n)) + !******************************************************************* + ! Calculate scaling function - see Eq 3 in Zarzycki (2016) + if (RSO_slab_option==0) RSO_X_cool = 1._r8/(1._r8+EXP(-0.5_r8*(u10-30._r8)) ) ! SLAB1 + if (RSO_slab_option==1) RSO_X_cool =(1._r8/(1._r8+EXP(-0.2_r8*(u10-30._r8)) ))*(u10*2.4_r8/80._r8) ! SLAB2 + if (RSO_slab_option==2) RSO_X_cool = 0.0_r8 ! THERMO + !******************************************************************* + ! compute new ocean surface temperature + o2x%rAttr(kt,n) = somtp(n) & + ! Thermodynamic terms + +( x2o%rAttr(kswnet,n) & ! shortwave net + +x2o%rAttr(klwup ,n) & ! longwave up + +x2o%rAttr(klwdn ,n) & ! longwave down + +x2o%rAttr(ksen ,n) & ! sfc sensible heat flux + +x2o%rAttr(klat ,n) & ! sfc latent heat flux + -x2o%rAttr(ksnow ,n)*latice & ! latent heat from snow + -x2o%rAttr(krofi ,n)*latice & ! latent heat from runoff + ) * dt/(cpsw*rhosw*hn) & + - RSO_X_cool*RSO_R_cool*((somtp(n)-RSO_Tdeep)/RSO_dT_o)*(RSO_h_o/hn)*dt & ! Turb mixing + + (1_r8/RSO_relax_tau)*(RSO_bckgrd_sst - somtp(n))*dt ! Newtonian Relaxation + !******************************************************************* + ! Ignore ice formed or melt potential + o2x%rAttr(kq,n) = 0.0 + ! Cap SSTs to freezing + o2x%rAttr(kt,n) = max( TkFrzSw, o2x%rAttr(kt,n) ) + ! Save temperature to send back to coupler + somtp(n) = o2x%rAttr(kt,n) + endif ! imask /= 0 + enddo ! lsize + endif ! firstcall + case('SOM_AQUAP') lsize = mct_avect_lsize(o2x) do n = 1,SDOCN%nstreams @@ -912,7 +991,10 @@ subroutine docn_comp_run(EClock, x2o, o2x, & close(nu) call shr_file_freeUnit(nu) endif - if (trim(datamode) == 'SOM' .or. trim(datamode) == 'SOM_AQUAP') then + if ( trim(datamode) == 'SOM' & ! Traditional slab ocean + .or. trim(datamode) == 'RSO' & ! Relaxed slab ocean + .or. trim(datamode) == 'SOM_AQUAP' & ! Aquaplanet slab ocean + ) then if (my_task == master_task) write(logunit,F04) ' writing ',trim(rest_file),target_ymd,target_tod call shr_pcdf_readwrite('write', SDOCN%pio_subsystem, SDOCN%io_type,& trim(rest_file), mpicom, gsmap, clobber=.true., rf1=somtp,rf1n='somtp') @@ -984,6 +1066,7 @@ subroutine prescribed_sst(xc, yc, lsize, sst_option, sst) integer :: i real(r8) :: tmp, tmp1, pi real(r8) :: rlon(lsize), rlat(lsize) + real(r8) :: mean_SST, delta_SST real(r8), parameter :: pio180 = SHR_CONST_PI/180._r8 @@ -1013,8 +1096,8 @@ subroutine prescribed_sst(xc, yc, lsize, sst_option, sst) ! Control - if (sst_option < 1 .or. sst_option > 10) then - call shr_sys_abort ('prescribed_sst: ERROR: sst_option must be between 1 and 10') + if (sst_option < 1 .or. sst_option > 15) then + call shr_sys_abort ('prescribed_sst: ERROR: sst_option must be between 1 and 15') end if if (sst_option == 1 .or. sst_option == 6 .or. sst_option == 7 .or. sst_option == 8) then @@ -1174,6 +1257,20 @@ subroutine prescribed_sst(xc, yc, lsize, sst_option, sst) end do end if + !------------------------------------------------------------------------------- + ! RCEMIP phase 2 - Mock-Walker + if (sst_option>=11 .and. sst_option<=15) then + if (sst_option==11) then; mean_SST = 295 - TkFrz; delta_SST = 1.250; end if ! MW_295dT1p25 + if (sst_option==12) then; mean_SST = 300 - TkFrz; delta_SST = 0.625; end if ! MW_300dT0p625 + if (sst_option==13) then; mean_SST = 300 - TkFrz; delta_SST = 1.250; end if ! MW_300dT1p25 + if (sst_option==14) then; mean_SST = 300 - TkFrz; delta_SST = 2.500; end if ! MW_300dT2p5 + if (sst_option==15) then; mean_SST = 305 - TkFrz; delta_SST = 1.250; end if ! MW_305dT1p25 + do i = 1, lsize + sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) + end do + end if + !------------------------------------------------------------------------------- + end subroutine prescribed_sst end module docn_comp_mod diff --git a/components/data_comps/docn/src/docn_shr_mod.F90 b/components/data_comps/docn/src/docn_shr_mod.F90 index cf59dbc7f620..fef714c83b4d 100644 --- a/components/data_comps/docn/src/docn_shr_mod.F90 +++ b/components/data_comps/docn/src/docn_shr_mod.F90 @@ -35,6 +35,8 @@ module docn_shr_mod character(CL) , public :: datamode ! mode integer(IN) , public :: aquap_option real(R8) , public :: sst_constant_value + real(R8) , public :: RSO_relax_tau ! relaxed slab ocean relaxation timescale [sec] + real(R8) , public :: RSO_fixed_MLD ! relaxed slab ocean globally fixed mixed layer depth (MLD) character(len=*), public, parameter :: nullstr = 'undefined' !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ CONTAINS @@ -77,7 +79,8 @@ subroutine docn_shr_read_namelists(mpicom, my_task, master_task, & !----- define namelist ----- namelist / docn_nml / & - decomp, restfilm, restfils, force_prognostic_true, sst_constant_value + decomp, restfilm, restfils, force_prognostic_true, sst_constant_value, & + RSO_fixed_MLD, RSO_relax_tau !---------------------------------------------------------------------------- ! Determine input filenamname @@ -110,12 +113,16 @@ subroutine docn_shr_read_namelists(mpicom, my_task, master_task, & write(logunit,F00)' restfils = ',trim(restfils) write(logunit,F0L)' force_prognostic_true = ',force_prognostic_true write(logunit,*) ' sst_constant_value = ',sst_constant_value + write(logunit,*) ' RSO_fixed_MLD = ',RSO_fixed_MLD + write(logunit,*) ' RSO_relax_tau = ',RSO_relax_tau endif call shr_mpi_bcast(decomp ,mpicom,'decomp') call shr_mpi_bcast(restfilm,mpicom,'restfilm') call shr_mpi_bcast(restfils,mpicom,'restfils') call shr_mpi_bcast(force_prognostic_true,mpicom,'force_prognostic_true') call shr_mpi_bcast(sst_constant_value ,mpicom,'sst_constant_value') + call shr_mpi_bcast(RSO_fixed_MLD ,mpicom,'RSO_fixed_MLD') + call shr_mpi_bcast(RSO_relax_tau ,mpicom,'RSO_relax_tau') rest_file = trim(restfilm) rest_file_strm = trim(restfils) @@ -152,8 +159,9 @@ subroutine docn_shr_read_namelists(mpicom, my_task, master_task, & trim(datamode) == 'SST_AQUAP_CONSTANT' .or. & trim(datamode) == 'COPYALL' .or. & trim(datamode) == 'IAF' .or. & - trim(datamode) == 'SOM' .or. & - trim(datamode) == 'SOM_AQUAP') then + trim(datamode) == 'SOM' .or. & ! Traditional slab ocean + trim(datamode) == 'RSO' .or. & ! Relaxed slab ocean + trim(datamode) == 'SOM_AQUAP') then ! Aquaplanet slab ocean if (my_task == master_task) then write(logunit,F00) ' docn datamode = ',trim(datamode) end if @@ -181,9 +189,9 @@ subroutine docn_shr_read_namelists(mpicom, my_task, master_task, & ocn_prognostic = .true. ocnrof_prognostic = .true. endif - if (trim(datamode) == 'SOM' .or. trim(datamode) == 'SOM_AQUAP') then - ocn_prognostic = .true. - endif + if (trim(datamode) == 'SOM') ocn_prognostic = .true. ! Traditional slab ocean + if (trim(datamode) == 'RSO') ocn_prognostic = .true. ! Relaxed slab ocean + if (trim(datamode) == 'SOM_AQUAP') ocn_prognostic = .true. ! Aquaplanet slab ocean end subroutine docn_shr_read_namelists diff --git a/components/data_comps/docs/index.md b/components/data_comps/docs/index.md new file mode 100644 index 000000000000..adef08426737 --- /dev/null +++ b/components/data_comps/docs/index.md @@ -0,0 +1,13 @@ +# Data Models + +The E3SM data models are key components to support many different scenarios: + +- AMIP style experiments with observed sea surface temperatures +- component model spin-up, such as land forced by atmospheric reanalysis +- idealized experiments, such as aquaplanet or radiative-convective equilibrium + +More details can be found for each component below. + +- [Data Atmosphere](user-guide/data-atmos.md) +- [Data Land](user-guide/data-land.md) +- [Data Ocean](user-guide/data-ocean.md) diff --git a/components/data_comps/docs/user-guide/data-atmos.md b/components/data_comps/docs/user-guide/data-atmos.md new file mode 100644 index 000000000000..d74710598ee9 --- /dev/null +++ b/components/data_comps/docs/user-guide/data-atmos.md @@ -0,0 +1,4 @@ +# The E3SM Data Atmosphere Model + +!!!WARNING + This page is still under construction diff --git a/components/data_comps/docs/user-guide/data-land.md b/components/data_comps/docs/user-guide/data-land.md new file mode 100644 index 000000000000..3b07f59793ec --- /dev/null +++ b/components/data_comps/docs/user-guide/data-land.md @@ -0,0 +1,4 @@ +# The E3SM Data Land Model + +!!!WARNING + This page is still under construction diff --git a/components/data_comps/docs/user-guide/data-ocean.md b/components/data_comps/docs/user-guide/data-ocean.md new file mode 100644 index 000000000000..de232d665e84 --- /dev/null +++ b/components/data_comps/docs/user-guide/data-ocean.md @@ -0,0 +1,123 @@ +# The E3SM Data Ocean Model + + + + +The E3SM data ocean has several different modes to support various realistic and idealized experiments. Sea surface temperatures (SST) can be either prescribed or prognostic. Prescribed SSTs are specified either through a data stream or analytically. Prognostic modes allow the SST field to evolve and respond to atmospheric conditions. The guides below provide more details on how to use these capabilities. + +- Prescribed + - [SST from Observations](#sst-from-observations) + - [Idealized SST](#idealized-sst) +- Prognostic + - [Traditional Slab Ocean Model](#traditional-slab-ocean-model) (SOM) + - [Relaxed Slab Ocean](#relaxed-slab-ocean) (RSO) + +## SST from Observations + +Using SST data derived from observations is the most common use of the data ocean model, often for AMIP style experiments to reproduce historical periods. + +Example compsets that use this capability are `F2010` and `F20TR`. These compsets use the `_DOCN%DOM_` compset modifier, which sets the `DOCN_MODE` variable in `env_run.xml` to "prescribed". + +Several additional XML variables need to be set in order to use this capability, which are set to defaults for common configurations, such as `F2010` at `ne30pg2` atmospheric resolution. + +```text +SSTICE_DATA_FILENAME Prescribed SST and ice coverage data file name +SSTICE_GRID_FILENAME Grid file in "domain" format corresponding to SSTICE_DATA_FILENAME +SSTICE_YEAR_ALIGN The model year that corresponds to SSTICE_YEAR_START on the data file +SSTICE_YEAR_START The first year of data to use from SSTICE_DATA_FILENAME +SSTICE_YEAR_END The last year of data to use from SSTICE_DATA_FILENAME +``` + +Most users will not need to edit these values from their defaults, but many scenarios require non-standard SST data, such as tropical cyclone hindcasts where the daily evolution of high-resolution SST data may be desireable. + +## Idealized SST + +The two main uses of idealized SST modes are aquaplanet (AQP) and radiative-convective equilibrium (RCE). The latter is just a special case of an aquaplanet where the SST is [usually] a constant value everywhere, traditionally used in conjunction with special modifications to homogenize radiation and disable rotation. There are several analytically specified SST patterns established by model intercomparison projects such as the Aqua-Planet Experiment (APE)[@blackburn_APE_context_2013] and RCEMIP[@wing_rcemip1_2018,@wing_rcemip2_2024]. + +### Idealized SST compsets + +The following list shows the currently defined E3SM compsets that utilize idealized SST. + +```text +FAQP +FAQP-MMF1 +FAQP-MMF2 +F-SCREAM-LR-AQP1 +F-SCREAM-HR-AQP1 +FRCE +FRCE-MMF1 +FRCE-MMF2 +FRCE-MW_295dT1p25 +FRCE-MW_300dT0p625 +FRCE-MW_300dT1p25 +FRCE-MW_300dT2p5 +FRCE-MW_305dT1p25 +FRCE-MW-MMF1_295dT1p25 +FRCE-MW-MMF1_300dT0p625 +FRCE-MW-MMF1_300dT1p25 +FRCE-MW-MMF1_300dT2p5 +FRCE-MW-MMF1_305dT1p25 +``` + +These all use "analytic" SST patterns that are specified via the `docn_comp_run()` subroutine in `components/data_comps/docn/src/docn_comp_mod.F90`. The `AQP` compsets currently only use the basic aquaplanet pattern that is symmetric about the equator. Other APE patterns introduce different meridional gradients and/or asymmetries. The various analytic SST patterns can be selected by changing the data ocean specifier: `_DOCN%AQP1_`. + +The first 10 analytic aquaplanet SST patterns correspond to the aqua-planet experiment (APE) protocol as follows + +```text +AQP1 = control symmetric SST pattern +AQP2 = Flat +AQP3 = Qobs = average of AQP1 and AQP2 +AQP4 = Peaked +AQP5 = Control+5N +AQP6 = 1KEQ - small warm pool +AQP7 = 3KEQ - small warm pool +AQP8 = 3KW1 - large warm pool +AQP9 = Control+10N +AQP10 = Control+15N +``` + +!!!NOTE + When using aquaplanet mode the orbital parameters will take on the idealized values shown below such that there are no seasonal variations, but there is still a diurnal cycle. + ```text + orb_eccen = 0 + orb_obliq = 0 + orb_mvelp = 0 + orb_mode = "fixed_parameters" + ``` + +The basic RCE compsets use the `_DOCN%AQPCONST_` modifier to produce a globally constant SST value, which is set by the `DOCN_AQPCONST_VALUE` variable in `env_run.xml`. The "FRCE-MW" compsets were designed for RCEMIP-II to produce a "mock walker-cell" configuration, in which sinusoidal SST variations are applied to promote a coherent large-scale circulation. + +### SST Data File + +In addition to the analytic SST modes the user can also specify an idealized aquaplanet SST pattern via the `_DOCN%AQPFILE_` option. The `aquapfile` namelist variable is used to specify the SST pattern in this mode. Note that this option has not been used or tested recently, so the user may experience difficulty trying to use this feature. + +## Traditional Slab Ocean Model + +A slab ocean model (SOM) allows responsive SSTs to address the "infinite heat source" problem associated with prescribed SSTs, but is much cheaper than running with a full ocean model. The traditional SOM appraoch requires special inputs, such as a specified mixed layer depth pattern that can vary in time and a prescribed heat flux to account for the missing effects of ocean dynamics often referred to as "Q-flux". The Q-flux data is often estimated from a fully coupled simulation with active ocean and sea-ice so that the SOM simulation will resemble the full model. + +Currently, we do not have Q-flux data to drive the SOM in E3SM. An alternative appraoch is to use a "relaxed" slab ocean (RSO) in which a specified relaxation time scale is used to bring the SST field back to a target SST field. The RSO mode is much simpler to use, but carries caveats that the user should be aware of before using. See [Data Ocean - Relaxed Slab Ocean](#relaxed-slab-ocean) for more information. + +## Relaxed Slab Ocean + +The relaxed slab ocean (RSO) is similar in many ways to the [traditional slab ocean model](#traditional-slab-ocean-model), but uses a specified relaxation time scale to avoid the need for specified "Q-flux" data to represent the effects of ocean transport. The RSO implementation in E3SM was inspired by Zarzycki (2016)[@Zarzycki_TC-ocn-cpl_2016]. + +A key consideration for the user is whether they need to use a realistic distribution of mixed layer depths (MLD), or whether their use case can benefit from the simplicity of a globally uniform MLD. + +The RSO mode has the following namelist variables to influence the ocean behavior: + +```text +RSO_relax_tau SST relaxation timescale +RSO_fixed_MLD globally uniform MLD value (use -1 for realistic MLD) +``` + +Other RSO parameter values are hardcoded in `components/data_comps/docn/src/docn_comp_mod.F90`. + +```text +RSO_slab_option = 0 ! Option for setting RSO_X_cool +RSO_R_cool = 11.75/86400 ! base cooling rate [K/s] +RSO_Tdeep = 271.00 ! deep water temperature [K] +RSO_dT_o = 27.0 ! scaling temperature gradient +RSO_h_o = 30.0 ! scaling mixed layer depth +``` + +The RSO mode uses the `SSTICE_DATA_FILENAME` in `env_run.xml` for its data stream. For a globally uniform MLD this file only need to contain a `SST_cpl` variable for the SST that will act as the target SST value for relaxation. If a realistic MLD pattern is desired then the `hblt` variable must also be present. This data can be derived a number of ways, but we currently do not have a dedicated tool or workflow. diff --git a/components/data_comps/drof/cime_config/config_component.xml b/components/data_comps/drof/cime_config/config_component.xml index b5e0a8071fab..1c0052d93644 100644 --- a/components/data_comps/drof/cime_config/config_component.xml +++ b/components/data_comps/drof/cime_config/config_component.xml @@ -13,23 +13,14 @@ --> - Data runoff model + Data runoff model NULL mode COREv2 normal year forcing: - COREv2 normal year forcing: - COREv2 normal year forcing: - COREv2 normal year forcing: COREv2 interannual year forcing: - COREv2 interannual year forcing: - COREv2 interannual year forcing: - COREv2 interannual year forcing: CPLHIST mode: + JRA55 interannual forcing, v1.5, through 2023 JRA55 interannual forcing, v1.5, through 2020 - JRA55 interannual forcing, v1.5, through 2020, no rofi or rofl around AIS JRA55 interannual forcing, v1.4, through 2018 - JRA55 interannual forcing, v1.4, through 2018, no rofi around AIS - JRA55 interannual forcing, v1.4, through 2018, no rofl around AIS - JRA55 interannual forcing, v1.4, through 2018, no rofi or rofl around AIS JRA55 interannual forcing JRA55 Repeat Year Forcing v1.3 1984-1985 JRA55 Repeat Year Forcing v1.3 1990-1991 @@ -47,26 +38,17 @@ char - CPLHIST,DIATREN_ANN_RX1,DIATREN_ANN_AIS00_RX1,DIATREN_ANN_AIS45_RX1,DIATREN_ANN_AIS55_RX1,DIATREN_IAF_RX1,DIATREN_IAF_AIS00_RX1,DIATREN_IAF_AIS45_RX1,DIATREN_IAF_AIS55_RX1,IAF_JRA,IAF_JRA_1p5,IAF_JRA_1p5_AIS0ROF,IAF_JRA_1p4_2018,IAF_JRA_1p4_2018_AIS0ICE,IAF_JRA_1p4_2018_AIS0LIQ,IAF_JRA_1p4_2018_AIS0ROF,RYF8485_JRA,RYF9091_JRA,RYF0304_JRA,NULL + CPLHIST,DIATREN_ANN_RX1,DIATREN_IAF_RX1,IAF_JRA,IAF_JRA_1p5,IAF_JRA_1p4_2018,RYF8485_JRA,RYF9091_JRA,RYF0304_JRA,NULL DIATREN_ANN_RX1 NULL DIATREN_ANN_RX1 - DIATREN_ANN_AIS00_RX1 - DIATREN_ANN_AIS45_RX1 - DIATREN_ANN_AIS55_RX1 DIATREN_IAF_RX1 - DIATREN_IAF_AIS00_RX1 - DIATREN_IAF_AIS45_RX1 - DIATREN_IAF_AIS55_RX1 CPLHIST IAF_JRA + IAF_JRA_1p5 IAF_JRA_1p5 - IAF_JRA_1p5_AIS0ROF IAF_JRA_1p4_2018 - IAF_JRA_1p4_2018_AIS0ICE - IAF_JRA_1p4_2018_AIS0LIQ - IAF_JRA_1p4_2018_AIS0ROF RYF8485_JRA RYF9091_JRA RYF0304_JRA @@ -165,6 +147,7 @@ 1 1 1 + 1 run_component_drof env_run.xml @@ -179,6 +162,7 @@ 1958 1958 1958 + 1958 run_component_drof env_run.xml @@ -193,6 +177,7 @@ 2016 2018 2020 + 2023 run_component_drof env_run.xml diff --git a/components/data_comps/drof/cime_config/namelist_definition_drof.xml b/components/data_comps/drof/cime_config/namelist_definition_drof.xml index c4139552c706..d4d70074596a 100644 --- a/components/data_comps/drof/cime_config/namelist_definition_drof.xml +++ b/components/data_comps/drof/cime_config/namelist_definition_drof.xml @@ -52,18 +52,8 @@ NULL rof.cplhist rof.diatren_ann_rx1 - rof.diatren_ann_ais00_rx1 - rof.diatren_ann_ais45_rx1 - rof.diatren_ann_ais55_rx1 rof.diatren_iaf_rx1 - rof.diatren_iaf_ais00_rx1 - rof.diatren_iaf_ais45_rx1 - rof.diatren_iaf_ais55_rx1 - rof.iaf_jra_1p4_2018_ais0ice - rof.iaf_jra_1p4_2018_ais0liq - rof.iaf_jra_1p4_2018_ais0rof rof.iaf_jra_1p4_2018 - rof.iaf_jra_1p5_ais0rof rof.iaf_jra_1p5 rof.iaf_jra rof.ryf8485_jra @@ -80,13 +70,7 @@ $DIN_LOC_ROOT/lnd/dlnd7/RX1 $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 $DIN_LOC_ROOT/lnd/dlnd7/JRA55 $DIN_LOC_ROOT/lnd/dlnd7/JRA55 $DIN_LOC_ROOT/lnd/dlnd7/JRA55 @@ -101,13 +85,7 @@ Stream domain file path(s). runoff.daitren.annual.20190226.nc - runoff.daitren.annual-AISx00.20190226.nc - runoff.daitren.annual-AISx45.20190226.nc - runoff.daitren.annual-AISx55.20190226.nc runoff.daitren.iaf.20120419.nc - runoff.daitren.iaf-AISx00.20120419.nc - runoff.daitren.iaf-AISx45.20120419.nc - runoff.daitren.iaf-AISx55.20120419.nc domain.roff.JRA025.170111.nc domain.roff.JRA025.170111.nc null @@ -134,27 +112,6 @@ arear area mask mask - - time time - xc lon - yc lat - arear area - mask mask - - - time time - xc lon - yc lat - arear area - mask mask - - - time time - xc lon - yc lat - arear area - mask mask - time time domrb_lon lon @@ -173,13 +130,7 @@ $DIN_LOC_ROOT/lnd/dlnd7/RX1 $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 $DIN_LOC_ROOT/ocn/jra55/v1.5_noleap $DIN_LOC_ROOT/lnd/dlnd7/JRA55 $DIN_LOC_ROOT/lnd/dlnd7/JRA55 @@ -194,31 +145,13 @@ Stream data file path(s). runoff.daitren.annual.20190226.nc - runoff.daitren.annual.20190226.nc - runoff.daitren.annual.20190226.nc - runoff.daitren.annual.20190226.nc runoff.daitren.iaf.20120419.nc - runoff.daitren.iaf-AISx00.20120419.nc - runoff.daitren.iaf-AISx45.20120419.nc - runoff.daitren.iaf-AISx55.20120419.nc RAF_8485.JRA.v1.3.runoff.180404.nc RAF_9091.JRA.v1.3.runoff.180404.nc RAF_0304.JRA.v1.3.runoff.180404.nc - - JRA.v1.5.runoff.%y.no_rofi_no_rofl.240411.nc - JRA.v1.5.runoff.%y.240411.nc - - JRA.v1.4.runoff.%y.no_rofi.190214.nc - - - JRA.v1.4.runoff.%y.no_rofl.190214.nc - - - JRA.v1.4.runoff.%y.no_rofi_no_rofl.190214.nc - JRA.v1.4.runoff.%y.190214.nc @@ -244,9 +177,6 @@ runoff rofl - - runoff rofl - rofl rofl rofi rofi @@ -282,7 +212,6 @@ 1 1 1 - 1 $DROF_STRM_YR_ALIGN $DROF_STRM_YR_ALIGN $DROF_CPLHIST_YR_ALIGN @@ -296,13 +225,7 @@ First year of stream. 1 - 1 - 1 - 1 1948 - 1948 - 1948 - 1948 $DROF_STRM_YR_START 1984 1990 @@ -318,18 +241,9 @@ Last year of stream. 1 - 1 - 1 - 1 2009 - 2009 - 2009 - 2009 $DROF_STRM_YR_END $DROF_STRM_YR_END - $DROF_STRM_YR_END - $DROF_STRM_YR_END - $DROF_STRM_YR_END $DROF_STRM_YR_END 1984 1990 diff --git a/components/data_comps/mkdocs.yml b/components/data_comps/mkdocs.yml new file mode 100644 index 000000000000..89f5e6dca7bf --- /dev/null +++ b/components/data_comps/mkdocs.yml @@ -0,0 +1,7 @@ +site_name: Data-Models + +nav: + - Introduction: 'index.md' + - Atmosphere: user-guide/data-atmos.md + - Land: user-guide/data-land.md + - Ocean: user-guide/data-ocean.md diff --git a/components/eam/bld/build-namelist b/components/eam/bld/build-namelist index 8dc532b6a3df..ee450573c367 100755 --- a/components/eam/bld/build-namelist +++ b/components/eam/bld/build-namelist @@ -830,6 +830,10 @@ add_default($nl,'use_hetfrz_classnuc'); add_default($nl,'hist_hetfrz_classnuc'); add_default($nl,'gw_convect_hcf') if (get_default_value('gw_convect_hcf')); add_default($nl,'hdepth_scaling_factor') if (get_default_value('hdepth_scaling_factor')); +add_default($nl,'gw_convect_hdepth_min') if (get_default_value('gw_convect_hdepth_min')); +add_default($nl,'gw_convect_storm_speed_min') if (get_default_value('gw_convect_storm_speed_min')); +add_default($nl,'gw_convect_plev_src_wind') if (get_default_value('gw_convect_plev_src_wind')); +add_default($nl,'use_gw_convect_old', 'val'=>'.true.'); add_default($nl,'linoz_psc_T'); if ($cfg->get('microphys') =~ /^mg2/) { add_default($nl,'micro_mg_dcs_tdep'); @@ -4089,13 +4093,24 @@ if ($waccm_phys or $cfg->get('nlev') >= 60) { add_default($nl, 'use_gw_oro' , 'val'=>'.true.'); add_default($nl, 'use_gw_front' , 'val'=>'.true.'); add_default($nl, 'use_gw_convect', 'val'=>'.true.'); + add_default($nl, 'use_od_ls', 'val'=>'.false.'); + add_default($nl, 'use_od_bl', 'val'=>'.false.'); + add_default($nl, 'use_od_ss', 'val'=>'.false.'); + add_default($nl, 'use_od_fd', 'val'=>'.false.'); } else { add_default($nl, 'use_gw_oro' , 'val'=>'.true.'); add_default($nl, 'use_gw_front' , 'val'=>'.false.'); add_default($nl, 'use_gw_convect', 'val'=>'.false.'); + add_default($nl, 'use_od_ls', 'val'=>'.false.'); + add_default($nl, 'use_od_bl', 'val'=>'.false.'); + add_default($nl, 'use_od_ss', 'val'=>'.false.'); + add_default($nl, 'use_od_fd', 'val'=>'.false.'); } add_default($nl, 'pgwv', 'val'=>'32'); add_default($nl, 'gw_dc','val'=>'2.5D0'); +add_default($nl, 'od_ls_ncleff' ,'val'=>'3.D0'); +add_default($nl, 'od_bl_ncd' ,'val'=>'3.D0'); +add_default($nl, 'od_ss_sncleff','val'=>'1.D0'); if ($nl->get_value('use_gw_oro') =~ /$TRUE/io) { add_default($nl, 'effgw_oro'); diff --git a/components/eam/bld/config_files/horiz_grid.xml b/components/eam/bld/config_files/horiz_grid.xml index 9053a492cda0..be2f8db26426 100644 --- a/components/eam/bld/config_files/horiz_grid.xml +++ b/components/eam/bld/config_files/horiz_grid.xml @@ -69,6 +69,7 @@ + diff --git a/components/eam/bld/namelist_files/namelist_defaults_eam.xml b/components/eam/bld/namelist_files/namelist_defaults_eam.xml index cfd9bf682c8e..7c1a7cd3fd51 100755 --- a/components/eam/bld/namelist_files/namelist_defaults_eam.xml +++ b/components/eam/bld/namelist_files/namelist_defaults_eam.xml @@ -107,6 +107,7 @@ +atm/cam/inic/homme/HICCUP.atm_era5.2015-01-01.ne0np4_CAx32v1_highorder.L128.nc atm/cam/inic/homme/cami-mam3_0000-01-01_arm30x8_L30_c130424.nc atm/cam/inic/homme/cami_0003-01-01_arm30x8_L26_ape_c000000.nc atm/cam/inic/homme/cami-mam3_0000-01-01_arm_x8v3_lowcon_np4_L30_c000000.nc @@ -126,7 +127,6 @@ atm/cam/topo/USGS-gtopo30_64x128_c050520.nc - atm/cam/topo/USGS-gtopo30_ne4np4_16x.c20160612.nc atm/cam/topo/USGS-gtopo30_ne4np4pg2_16x_converted.c20200527.nc atm/cam/topo/USGS-gtopo30_ne11np4_16xconsistentSGH.c20160612.nc @@ -148,6 +148,7 @@ atm/cam/topo/USGS-gtopo30_ne512np4pg2_16xconsistentSGH_20190212_converted.nc atm/cam/topo/USGS-gtopo30_ne1024np4pg2_16xconsistentSGH_20190528_converted.nc +atm/cam/topo/USGS-gtopo30_CA_ne32_x32_v1_pg2_16xdel2.nc atm/cam/inic/homme/USGS-gtopo30_arm_x8v3_lowcon_tensor12xconsistentSGH.nc atm/cam/topo/USGS_conusx4v1-tensor12x_consistentSGH_c150924.nc atm/cam/topo/USGS_conusx4v1pg2_12x_consistentSGH_20200609.nc @@ -723,6 +724,7 @@ atm/cam/chem/trop_mam/atmsrf_ne512pg2_200212.nc atm/cam/chem/trop_mam/atmsrf_ne1024pg2_200212.nc +atm/cam/chem/trop_mam/atmsrf_CAx32v1_pg2_20220517.nc atm/cam/chem/trop_mam/atmsrf_armx8v3.nc atm/cam/chem/trop_mam/atmsrf_conusx4v1.nc atm/cam/chem/trop_mam/atmsrf_conusx4v1pg2_20020609.nc @@ -824,7 +826,7 @@ 0.5D0 7.0D0 0.001D0 - 0.04D0 + 0.08D0 2.65D0 0.02D0 0.1D0 @@ -1420,7 +1422,7 @@ - + 0 0 0 0 @@ -1430,6 +1432,7 @@ 0 /dev/null +atm/cam/inic/homme/CA_ne32_x32_v1.g atm/cam/inic/homme/arm_x8v3_lowcon.g atm/cam/inic/homme/conusx4v1.g atm/cam/inic/homme/northamericax4v1.g @@ -1444,6 +1447,7 @@ 0d0 2.5e5 + 1e4 1e5 4.3e4 4e4 @@ -1513,6 +1517,10 @@ 6 6 + 8.333333333333333d0 + 6 + 6 + 3 3 3 @@ -1527,7 +1535,6 @@ 2 2 - 2 2 @@ -1882,8 +1889,14 @@ with se_tstep, dt_remap_factor, dt_tracer_factor set to -1 10.0 0.50 1.0 +2.5 +10.0 +70000 0.375 .true. + 3.D0 + 3.D0 + 1.D0 2.5D0 268.15D0 13.8D0 diff --git a/components/eam/bld/namelist_files/namelist_definition.xml b/components/eam/bld/namelist_files/namelist_definition.xml index 8228c7d8d2e3..848e43425b1b 100644 --- a/components/eam/bld/namelist_files/namelist_definition.xml +++ b/components/eam/bld/namelist_files/namelist_definition.xml @@ -1078,6 +1078,48 @@ Whether or not to enable GWD brute-force energy fix. Default: set by build-namelist. + +Whether or not to enable nonlinear orographic gravity wave drag (oGWD). +Default: set by build-namelist. + + + +Whether or not to enable flow-blocking drag (FBD). +Default: set by build-namelist. + + + +Whether or not to enable small-scale orographic GWD drag (sGWD). +Default: set by build-namelist. + + + +Whether or not to enable turbulent orographic form drag (TOFD). +Default: set by build-namelist. + + + +Tuning parameter of orographic GWD (oGWD). See use_od_ls. +Default: set by build-namelist. + + + +Tuning parameter of flow-blocking drag (FBD). See use_od_bl. +Default: set by build-namelist. + + + +Tuning parameter of small-scale GWD (sGWD). See use_od_ss. +Default: set by build-namelist. + + Gravity wave spectrum dimension (wave numbers are from -pgwv to pgwv). @@ -1100,16 +1142,40 @@ Default: set by build-namelist. -Heating rate conversion factor associated with convective gravity waves +Heating rate conversion factor associated with convective GWD [unitless]. +Also often interpretted as "convective fraction" [%]. Default: 20.0 -Scaling factor for the heating depth +Scaling factor for the heating depth [unitless] Default: 1.0 + +minimum hdepth for for convective GWD spectrum lookup table [km] +Default: 2.5 km + + + +minimum convective storm speed in m/s for convective GWD +Default: 10.0 m/s + + + +Reference pressure value used for convective storm speed in m/s for convective GWD +Default: 70000 Pa + + + +switch to revert to old calculation of Beres scheme for heating depth and max + + Efficiency associated with convective gravity waves from the Beres @@ -6743,6 +6809,36 @@ example, to apply hyperviscosity to moisture, the first tracer, set the value to Default: (set by dycore) + +Number of element haloes to include in trajectory search. -1 triggers an +automatic calculation of max(1, dt_tracer_factor/3). This is based on the +advective CFL condition that governs the dynamics time step. +Default: -1 (set by dycore) + + + +Number of substeps to take in computing semi-Lagrangian transport +trajectories. 0 triggers the original algorithm; 1 or larger triggers the +enhanced-trajectory algorithm. +Default: 0 (set by dycore) + + + +Number of velocity snapshots to store for use when computing the enhanced +trajectories. -1 triggers an automatic calculation. 0, 1, 2 all become 2, the +minimum. +Default: -1 (set by dycore) + + + +Optional diagnostic output from transport module. +Default: 0 (set by dycore) + + 0.0 - + diff --git a/components/eam/cime_config/buildnml b/components/eam/cime_config/buildnml index a86cd0fc17d9..beea61fd215f 100755 --- a/components/eam/cime_config/buildnml +++ b/components/eam/cime_config/buildnml @@ -233,29 +233,6 @@ def buildnml(case, caseroot, compname): safe_copy(os.path.join(eamconf_dir, "atm_in"), os.path.join(rundir, "atm_in{}".format(inst_string))) safe_copy(os.path.join(eamconf_dir, "drv_flds_in"), os.path.join(rundir, "drv_flds_in")) - # ----------------------------------------------------- - # copy scream input data - # ----------------------------------------------------- - - with SharedArea(): - scream_data_dir = os.path.join(case.get_value("SRCROOT"), "components/eamxx/data") - for item in os.listdir(scream_data_dir): - tgt_dir = os.path.join(din_loc_root, "atm/cam/physprops") - tgt_path = os.path.join(tgt_dir, item) - if not os.path.isdir(tgt_dir): - try: - os.makedirs(tgt_dir) - except OSError: - pass # lost the race - - try: - fd = os.open(tgt_path, os.O_CREAT | os.O_EXCL) - # If we get to this line, we won the race - os.close(fd) - safe_copy(os.path.join(scream_data_dir, item), tgt_path) - except OSError: - pass # lost the race - ############################################################################### def _main_func(): ############################################################################### diff --git a/components/eam/cime_config/config_compsets.xml b/components/eam/cime_config/config_compsets.xml index 7050bd0c8f90..f68d2f45e224 100644 --- a/components/eam/cime_config/config_compsets.xml +++ b/components/eam/cime_config/config_compsets.xml @@ -139,6 +139,15 @@ FAQP-MMF2 2000_EAM%AQP-MMF2_SLND_SICE_DOCN%AQP1_SROF_SGLC_SWAV + + + + + + + F2010-RSO + 2010_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI%PRES_DOCN%RSO_MOSART_SGLC_SWAV + @@ -229,6 +238,50 @@ 2000_EAM%RCE-MMF2_SLND_SICE_DOCN%AQPCONST_SROF_SGLC_SWAV + + + FRCE-MW_295dT1p25 + 2000_EAM%RCE_SLND_SICE_DOCN%AQP11_SROF_SGLC_SWAV + + + FRCE-MW_300dT0p625 + 2000_EAM%RCE_SLND_SICE_DOCN%AQP12_SROF_SGLC_SWAV + + + FRCE-MW_300dT1p25 + 2000_EAM%RCE_SLND_SICE_DOCN%AQP13_SROF_SGLC_SWAV + + + FRCE-MW_300dT2p5 + 2000_EAM%RCE_SLND_SICE_DOCN%AQP14_SROF_SGLC_SWAV + + + FRCE-MW_305dT1p25 + 2000_EAM%RCE_SLND_SICE_DOCN%AQP15_SROF_SGLC_SWAV + + + + FRCE-MW-MMF1_295dT1p25 + 2000_EAM%RCE-MMF1_SLND_SICE_DOCN%AQP11_SROF_SGLC_SWAV + + + FRCE-MW-MMF1_300dT0p625 + 2000_EAM%RCE-MMF1_SLND_SICE_DOCN%AQP12_SROF_SGLC_SWAV + + + FRCE-MW-MMF1_300dT1p25 + 2000_EAM%RCE-MMF1_SLND_SICE_DOCN%AQP13_SROF_SGLC_SWAV + + + FRCE-MW-MMF1_300dT2p5 + 2000_EAM%RCE-MMF1_SLND_SICE_DOCN%AQP14_SROF_SGLC_SWAV + + + FRCE-MW-MMF1_305dT1p25 + 2000_EAM%RCE-MMF1_SLND_SICE_DOCN%AQP15_SROF_SGLC_SWAV + + + diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/orodrag_ne30pg2/user_nl_eam b/components/eam/cime_config/testdefs/testmods_dirs/eam/orodrag_ne30pg2/user_nl_eam new file mode 100644 index 000000000000..8ab37d279783 --- /dev/null +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/orodrag_ne30pg2/user_nl_eam @@ -0,0 +1,8 @@ +use_gw_oro=.false. +use_od_ls=.true. +use_od_bl=.true. +use_od_ss=.true. +use_od_fd=.true. + + +bnd_topo='$DIN_LOC_ROOT/atm/cam/topo/USGS-gtopo30_ne30np4pg2_x6t-SGH_forOroDrag.c20241001.nc' diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/orodrag_ne4pg2/user_nl_eam b/components/eam/cime_config/testdefs/testmods_dirs/eam/orodrag_ne4pg2/user_nl_eam new file mode 100644 index 000000000000..f32cc8a6f936 --- /dev/null +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/orodrag_ne4pg2/user_nl_eam @@ -0,0 +1,7 @@ +use_gw_oro=.false. +use_od_ls=.true. +use_od_bl=.true. +use_od_ss=.true. +use_od_fd=.true. + +bnd_topo='$DIN_LOC_ROOT/atm/cam/topo/USGS-gtopo30_ne4np4pg2_16x_converted_forOroDrag.c20241019.nc' diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/readme b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/readme new file mode 100644 index 000000000000..35f64eb4b9a1 --- /dev/null +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/readme @@ -0,0 +1 @@ +test for sat hist capability components/eam/src/control/sat_hist.F90 diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/shell_commands b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/shell_commands new file mode 100644 index 000000000000..92cb057059a4 --- /dev/null +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/shell_commands @@ -0,0 +1,3 @@ +#!/bin/bash + +./xmlchange RUN_STARTDATE=2018-01-01 diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/user_nl_eam b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/user_nl_eam new file mode 100644 index 000000000000..e9b723f13b88 --- /dev/null +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/user_nl_eam @@ -0,0 +1,7 @@ +&satellite_options_nl + sathist_mfilt = 10000, + sathist_track_infile = '$DIN_LOC_ROOT/atm/waccm/sat/satellite_profilelist_orcas_to_socrates_c190208.nc' + sathist_hfilename_spec = '%c.eam.h9.sathist.%y-%m-%d-%s.nc' + sathist_nclosest = 1 + sathist_ntimestep = 1 + sathist_fincl = 'T', 'PS' diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_sl_nsubstep2/user_nl_eam b/components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_sl_nsubstep2/user_nl_eam new file mode 100644 index 000000000000..7cfe7f1cb6e2 --- /dev/null +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_sl_nsubstep2/user_nl_eam @@ -0,0 +1,3 @@ +semi_lagrange_diagnostics = 1 +semi_lagrange_trajectory_nsubstep = 2 +semi_lagrange_trajectory_nvelocity = 3 diff --git a/components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl b/components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl index 3ecd465f7a14..3a47a8bb2ff9 100644 --- a/components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl +++ b/components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl @@ -31,4 +31,5 @@ seq_flux_mct_albdif = 0.07 seq_flux_mct_albdir = 0.07 seq_flux_atmocn_minwind = 1 -constant_zenith_deg = 42.05 \ No newline at end of file +constant_zenith_deg = 42.04 + diff --git a/components/eam/docs/figures/orodrag.png b/components/eam/docs/figures/orodrag.png new file mode 100644 index 000000000000..de6e5cd84072 Binary files /dev/null and b/components/eam/docs/figures/orodrag.png differ diff --git a/components/eam/docs/tech-guide/index.md b/components/eam/docs/tech-guide/index.md index f93b77155705..10e669367997 100644 --- a/components/eam/docs/tech-guide/index.md +++ b/components/eam/docs/tech-guide/index.md @@ -16,6 +16,8 @@ This Technical Guide describes the physics of version 3 of the E3SM Atmospheric - [RRTMG](rrtmg.md): Parameterization of radiation. +- [ORODRAG](orodrag.md): Parameterization of orographic drag + - [MAM](mam.md): Primary parameterization schemes used to represent aerosols. - [VBS](vbs.md): Parameterization of secondary organic aerosols. diff --git a/components/eam/docs/tech-guide/orodrag.md b/components/eam/docs/tech-guide/orodrag.md new file mode 100644 index 000000000000..a0be96e4f756 --- /dev/null +++ b/components/eam/docs/tech-guide/orodrag.md @@ -0,0 +1,35 @@ +# Orographic drag scheme + +## Overview + +The orographic drag schemes includes two main options: the default Gravity Wave Drag scheme of McFarlane (1987)[@mcfarlane_the_1987] and a new suite of orographic drag parameterization schemes. The new suite includes 4 components all combined in one module (i.e. subroutine gwdo2d). The schemes include orographic gravity wave drag (oGWD, Xie et al.,2020)[@xie_an_2020], flow-blocking drag (FBD, Xie et al.,2020)[@xie_an_2020], small-scale GWD (sGWD, Tsiringakis et al.,2017)[@tsiringakis_small_2020], and turbulent-scale orographic form drag (TOFD, Beljaars et al., 2004)[@beljaars_a_2020]. The oGWD and TOFD schemes are used to replace the default oGWD (McFarlane, 1987)[@mcfarlane_the_1987] and Turbulent Mountain Stress (TMS, documented on Richter et al., 2010)[@richter_the_2010] module in EAM, while the FBD and sGWD are added enhanced drag schemes. Each of the schemes are used to predict and add enhanced drags from surface to high level that would help decelerate the wind especially over the mountainous regions among the globe. The oGWD, FBD, and sGWD are implemented in gw_drag.F90, while TOFD is implemented in clubb_intr.F90 to join the vertical diffusion process. There are also new topographic parameters (orographic asymmetry [OA], orographic convexity [OC], effective orographic length [OL]) are input into the model for the new oGWD and FBD scheme implmented. The concept of each scheme is illustrate in the figure below. Currently, only the scheme of McFarlane (1987) is turned on as default in E3SMv3.0. + +![orodrag figure](../figures/orodrag.png) + +### Default oGWD scheme + +The current default oGWD scheme in E3SMv3.0 is from McFarlane (1987)[@mcfarlane_the_1987]. It is a linear orographic gravity wave drag scheme that parameterizes the subgrid process of vertical propagation of gravity wave originating from orographic source. The output from this scheme is a vertical profile of drag (or deceleration terms) when gravity wave breaks at higher levels and deposits momemtum flux to that level. This scheme is shown to improve the excessive westerly wind bias in the extratropics and the wind bias in the polar region. This scheme is turned on by default in E3SMv3.0. + +### Default TMS scheme + +The turbulent mountain stress (TMS) scheme is documented on Richter et al.,(2010)[@richter_the_2010]. It parameterizes the turbulent scale form drag (TOFD) through adding enhanced effective roughness length to the model. This method is justified by the observation that area-averaged wind sufficiently far above hilly terrain behave logarithmically. It leads to significant decrease of overspeed surface wind in the model. It is turned OFF in E3SMv3.0. + +### New oGWD scheme + +The Xie et al.(2020)[@xie_an_2020] oGWD scheme is a nonlinear orographic gravity wave drag scheme that parameterizes the subgrid oGWD (>5km). It includes both the linear oGWD like that of McFarlane (1987)[@mcfarlane_the_1987] and nonlinear oGWD like low-level wave breaking (LLWB). The LLWB downstream often results in downslope windstorm that can enhance the drag by several times that of linear oGWD. The incorporation of the nonlinear drag is by adding an "enhancement factor" that is a function of the higher moments of the orography, e.g. orographic asymmetry (OA), orographic convexity (OC), effective orographic length (OL). + +### FBD scheme + +The Xie et al.(2020)[@xie_an_2020] FBD scheme parameterizes the drag by flow blocked on the mountain flanks or flowing around the mountain under upstream stable conditions (>5km). It occurs when the mean flow does not have enough kinetic energy to traverse an obstacle and either stops upstream or diverges around the obstacle. This provides drag near the surface where the blocking occurs in addition to the oGWD. + +### TOFD scheme + +The TOFD scheme (Beljaars et al.,2004)[@beljaars_a_2020] parameterizes the drag generated by the shear stresses in the boundary layer when the flow encounters smaller obstacles (<5km). As airflow encounters an obstacle, it is disrupted, leading to the formation of eddies and vortices. These turbulent structures enhance mixing, allowing different layers of air to interact over a short distance. The intensity of the mixing is typically higher close to the obstacle, with rapid changes in velocity and direction of the airflow and can exhibit large control over the surface wind. It is an alternative scheme to the current TMS in E3SMv3. An important difference between the TOFD and TMS is that TOFD explicitly calculate the stress profile while TMS uses an enhanced effective roughtness length approach. + +### sGWD scheme + +The Tsiringakis et al.(2017)[@tsiringakis_small_2020] sGWD scheme parameterizes the small-scale orographic gravity wave drag (sGWD) within the relatively shallow stable boundary layer. Models applying the form drag such as TOFD schemes usually lack sufficient cyclonic filling and do not accurately represent the development of cyclones over land (Holstag 2006)[@holtslag_preface_2006]. This caused significant temperature bias that occurs due to runaway cooling at the surface. To bridge the gap of this missing drag in the boundary layer, studies have hypothesized that the missing drag can be generated by sGWD. sGWD occurs under the relative stable boundary layer with low winds where the turbulence is significantly reduced. + +## Namelist parameters + +[orodrag Namelist Parameters](../user-guide/namelist_parameters.md#orographic-drag-schemes) diff --git a/components/eam/docs/user-guide/namelist_parameters.md b/components/eam/docs/user-guide/namelist_parameters.md index b09e8c44ffde..8aa4b40299c5 100644 --- a/components/eam/docs/user-guide/namelist_parameters.md +++ b/components/eam/docs/user-guide/namelist_parameters.md @@ -156,3 +156,20 @@ | Parameter | Description | Default value | | ------------------------- | ----------------------------------------------------------------- | ---------------------- | | `cosp_lite` | This namelist sets cosp_ncolumns=10 and cosp_nradsteps=3 (appropriate for COSP statistics derived from seasonal averages), and runs MISR, ISCCP, MODIS, and CALIPSO lidar simulators (cosp_lmisr_sim=.true.,cosp_lisccp_sim=.true., cosp_lmodis_sim=.true.,cosp_llidar_sim=.true.). | `false` | + +## Orographic drag schemes + +| Parameter | Description | Default value | +| ------------------------- | ----------------------------------------------------------------- | ---------------------- | +| `use_gw_oro` | This namelist controls the default linear orographic gravity wave drag (oGWD) for E3SM, if used, the default oGWD is turned on. | `true` | +| `do_tms` | This namelist controls the default Turbulent Mountain Stress (TMS) for E3SM, if used, the default TMS is turned on. | `false` | +| `effgw_oro` | Efficiency associated with orographic gravity waves. | `0.375` | +| `tms_orocnst` | Turbulent mountain stress parameter used when TMS calculation is turned on | `1.0` | +| `tms_z0fac` | Factor determining z_0 from orographic standard deviation [ no unit ] for TMS. | `0.75` | +| `use_od_ls` | This namelist controls the new nonlinear oGWD, if used, the nonlinear oGWD is turned on. use_od_ls should not be used at the same time with use_gw_oro. | `false` | +| `use_od_bl` | This namelist controls the Flow-blocking drag (FBD) scheme, if used, the FBD scheme is turned on. | `false` | +| `use_od_ss` | This namelist controls the small-scale GWD (sGWD) scheme, if used, the sGWD scheme is turned on. | `false` | +| `use_od_fd` | This namelist controls the Turbulent orographic form drag (TOFD) scheme, if used, the TOFD scheme is turned on. | `false` | +| `od_ls_ncleff` | Tuning parameter of nonlinear oGWD. Stands for effective resolution of the grid for oGWD. Scales the magnitude of nonlinear oGWD. | `3` | +| `od_bl_ncd` | Tuning parameter of FBD. Stands for bulk drag coefficient. Scales the magnitude of FBD. | `3` | +| `od_ss_sncleff` | Tuning parameter of sGWD. Stands for effective resolution of the grid for sGWD.Scales the magnitude of sGWD. | `1` | diff --git a/components/eam/mkdocs.yml b/components/eam/mkdocs.yml index e41cb614387d..d0e23a6e7e8a 100644 --- a/components/eam/mkdocs.yml +++ b/components/eam/mkdocs.yml @@ -16,6 +16,7 @@ nav: - tech-guide/clubb.md - tech-guide/zm.md - RRTMG: tech-guide/rrtmg.md + - tech-guide/orodrag.md - tech-guide/mam.md - tech-guide/vbs.md - tech-guide/dust.md diff --git a/components/eam/src/chemistry/modal_aero/modal_aero_amicphys.F90 b/components/eam/src/chemistry/modal_aero/modal_aero_amicphys.F90 index 6546677cac3b..5f7f690cf625 100644 --- a/components/eam/src/chemistry/modal_aero/modal_aero_amicphys.F90 +++ b/components/eam/src/chemistry/modal_aero/modal_aero_amicphys.F90 @@ -3530,7 +3530,7 @@ subroutine mam_soaexch_vbs_1subarea( & ! convert sat vapor conc from ug/m^3 to mol/m^3 then to mol/liter tmpa = (c0_soa_298(ll)*1.0e-6_r8/mw_gas(igas)) * 1.0e-3_r8 ! calc sat vapor pressure (atm) from molar-conc and temp [ 0.082056 = gas constant in (atm/deg-K/(mol/liter)) ] - p0_soa_298(ll) = 0.082056_r8*tmpa*temp + p0_soa_298(ll) = 0.082056_r8*tmpa*298.0_r8 end do ! calc soa gas saturation molar-mixing-ratio at local temp and air-pressure diff --git a/components/eam/src/chemistry/mozart/rate_diags.F90 b/components/eam/src/chemistry/mozart/rate_diags.F90 index 457ede703076..4ad21392f544 100644 --- a/components/eam/src/chemistry/mozart/rate_diags.F90 +++ b/components/eam/src/chemistry/mozart/rate_diags.F90 @@ -126,7 +126,7 @@ subroutine rate_diags_calc( rxt_rates, vmr, m, ncol, lchnk, pver, pdeldry, mbar rxt_rates(:ncol,:,rxt_tag_map(i)) = rxt_rates(:ncol,:,rxt_tag_map(i)) * m(:,:) call outfld( rate_names(i), rxt_rates(:ncol,:,rxt_tag_map(i)), ncol, lchnk ) - if ( .not. history_UCIgaschmbudget_2D .and. .not. history_UCIgaschmbudget_2D_levels) return + if (history_UCIgaschmbudget_2D .or. history_UCIgaschmbudget_2D_levels) then if (rate_names(i) .eq. 'r_lch4') then !kg/m2/sec @@ -166,6 +166,8 @@ subroutine rate_diags_calc( rxt_rates, vmr, m, ncol, lchnk, pver, pdeldry, mbar endif endif + + endif enddo end subroutine rate_diags_calc diff --git a/components/eam/src/control/runtime_opts.F90 b/components/eam/src/control/runtime_opts.F90 index c52c02a5c238..1fcd84806b77 100644 --- a/components/eam/src/control/runtime_opts.F90 +++ b/components/eam/src/control/runtime_opts.F90 @@ -246,6 +246,7 @@ subroutine read_namelist(single_column_in, scmlon_in, scmlat_in, scm_multcols_in use uwshcu, only: uwshcu_readnl use pkg_cld_sediment, only: cld_sediment_readnl use gw_drag, only: gw_drag_readnl + use od_common, only: oro_drag_readnl use qbo, only: qbo_readnl use iondrag, only: iondrag_readnl use phys_debug_util, only: phys_debug_readnl @@ -516,6 +517,7 @@ subroutine read_namelist(single_column_in, scmlon_in, scmlat_in, scm_multcols_in call uwshcu_readnl(nlfilename) call cld_sediment_readnl(nlfilename) call gw_drag_readnl(nlfilename) + call oro_drag_readnl(nlfilename) call qbo_readnl(nlfilename) call iondrag_readnl(nlfilename) call phys_debug_readnl(nlfilename) diff --git a/components/eam/src/control/sat_hist.F90 b/components/eam/src/control/sat_hist.F90 index 64b978ac0f16..17d2a92bc7bd 100644 --- a/components/eam/src/control/sat_hist.F90 +++ b/components/eam/src/control/sat_hist.F90 @@ -6,18 +6,20 @@ module sat_hist use perf_mod, only: t_startf, t_stopf - use shr_kind_mod, only: r8 => shr_kind_r8 + use shr_kind_mod, only: r4 => shr_kind_r4 + use shr_kind_mod, only: r8 => shr_kind_r8, cl=>shr_kind_cl use cam_logfile, only: iulog - use ppgrid, only: pcols, pver, begchunk, endchunk + use ppgrid, only: pcols, pver, pverp, begchunk, endchunk use cam_history_support, only: fieldname_lenp2, max_string_len, ptapes use spmd_utils, only: masterproc, iam use cam_abortutils, only: endrun - use pio, only: file_desc_t, iosystem_desc_t, iosystem_desc_t, var_desc_t, io_desc_t - use pio, only: pio_openfile, pio_redef, pio_enddef, pio_inq_dimid, pio_inq_varid, pio_seterrorhandling, pio_def_var + use pio, only: file_desc_t,iosystem_desc_t, var_desc_t, io_desc_t + use pio, only: pio_inq_dimid, pio_inq_varid + use pio, only: pio_seterrorhandling, pio_def_var use pio, only: pio_inq_dimlen, pio_get_att, pio_put_att, pio_get_var, pio_put_var, pio_write_darray - use pio, only: pio_real, pio_int, pio_double - use pio, only: PIO_WRITE,PIO_NOWRITE, PIO_NOERR, PIO_BCAST_ERROR, PIO_INTERNAL_ERROR, PIO_Rearr_box, PIO_GLOBAL + use pio, only: pio_real,pio_double + use pio, only: PIO_NOWRITE, PIO_NOERR, PIO_BCAST_ERROR, PIO_INTERNAL_ERROR, PIO_GLOBAL use spmd_utils, only: mpicom #ifdef SPMD use mpishorthand, only: mpichar, mpiint @@ -82,6 +84,7 @@ module sat_hist real(r8), parameter :: rad2deg = 180._r8/pi ! degrees per radian + contains !------------------------------------------------------------------------------- @@ -122,7 +125,7 @@ subroutine sat_hist_readnl(nlfile, hfilename_spec, mfilt, fincl, nhtfrq, avgflag ! set defaults sathist_track_infile = ' ' - sathist_hfilename_spec = '%c.cam' // trim(inst_suffix) // '.hs.%y-%m-%d-%s.nc' + sathist_hfilename_spec = '%c.eam.hs.' // trim(inst_suffix) // '.%y-%m-%d-%s.nc' sathist_fincl(:) = ' ' sathist_mfilt = 100000 sathist_nclosest = 1 @@ -189,14 +192,13 @@ end subroutine sat_hist_readnl subroutine sat_hist_init use cam_pio_utils, only: cam_pio_openfile use ioFileMod, only: getfil - use spmd_utils, only: npes use time_manager, only: get_step_size use string_utils, only: to_lower, GLC implicit none character(len=max_string_len) :: locfn ! Local filename - integer :: ierr, dimid, i + integer :: ierr, dimid character(len=128) :: date_format @@ -406,17 +408,15 @@ end subroutine sat_hist_define !------------------------------------------------------------------------------- subroutine sat_hist_write( tape , nflds, nfils) - use ppgrid, only : pcols, begchunk, endchunk use phys_grid, only: phys_decomp use dyn_grid, only: dyn_decomp use cam_history_support, only : active_entry - use pio, only : pio_file_is_open - implicit none + use pio, only : pio_file_is_open, pio_syncfile type(active_entry) :: tape integer, intent(in) :: nflds integer, intent(inout) :: nfils - integer :: t, f, i, ncols, nocols + integer :: ncols, nocols integer :: ierr integer, allocatable :: col_ndxs(:) @@ -430,9 +430,13 @@ subroutine sat_hist_write( tape , nflds, nfils) real(r8),allocatable :: phs_dists(:) integer :: coldim - - integer :: io_type - logical :: has_dyn_flds + logical :: has_dyn_flds = .false. + logical :: has_phys_srf_flds = .false. + logical :: has_phys_lev_flds = .false. + logical :: has_phys_ilev_flds = .false. + logical :: has_dyn_srf_flds = .false. + logical :: has_dyn_lev_flds = .false. + logical :: has_dyn_ilev_flds = .false. if (.not.has_sat_hist) return @@ -456,13 +460,11 @@ subroutine sat_hist_write( tape , nflds, nfils) allocate( mlons(nocols) ) allocate( phs_dists(nocols) ) - has_dyn_flds = .false. - dyn_flds_loop: do f=1,nflds - if ( tape%hlist(f)%field%decomp_type == dyn_decomp ) then - has_dyn_flds = .true. - exit dyn_flds_loop - endif - enddo dyn_flds_loop + call scan_flds( tape, nflds & + , has_phys_srf_flds, has_phys_lev_flds, has_phys_ilev_flds & + , has_dyn_srf_flds, has_dyn_lev_flds, has_dyn_ilev_flds ) + + has_dyn_flds = has_dyn_srf_flds .or. has_dyn_lev_flds .or. has_dyn_ilev_flds call get_indices( obs_lats, obs_lons, ncols, nocols, has_dyn_flds, col_ndxs, chk_ndxs, & fdyn_ndxs, ldyn_ndxs, phs_owners, dyn_owners, mlats, mlons, phs_dists ) @@ -479,16 +481,35 @@ subroutine sat_hist_write( tape , nflds, nfils) call write_record_coord( tape, mlats(:), mlons(:), phs_dists(:), ncols, nfils ) - do f=1,nflds + ! dump columns of 2D fields + if (has_phys_srf_flds) then + call dump_columns( tape%File, tape%hlist, nflds, nocols, 1, nfils, & + col_ndxs, chk_ndxs, phs_owners, phys_decomp ) + endif + if (has_dyn_srf_flds) then + call dump_columns( tape%File, tape%hlist, nflds, nocols, 1, nfils, & + fdyn_ndxs, ldyn_ndxs, dyn_owners, dyn_decomp ) + endif - select case (tape%hlist(f)%field%decomp_type) - case (phys_decomp) - call dump_columns(tape%File, tape%hlist(f), nocols, nfils, col_ndxs(:), chk_ndxs(:), phs_owners(:) ) - case (dyn_decomp) - call dump_columns(tape%File, tape%hlist(f), nocols, nfils, fdyn_ndxs(:), ldyn_ndxs(:), dyn_owners(:) ) - end select + ! dump columns of 3D fields defined on mid pres levels + if (has_phys_lev_flds) then + call dump_columns( tape%File, tape%hlist, nflds, nocols, pver, nfils, & + col_ndxs, chk_ndxs, phs_owners, phys_decomp ) + endif + if (has_dyn_lev_flds) then + call dump_columns( tape%File, tape%hlist, nflds, nocols, pver, nfils, & + fdyn_ndxs, ldyn_ndxs, dyn_owners, dyn_decomp ) + endif - enddo + ! dump columns of 3D fields defined on interface pres levels + if (has_phys_ilev_flds) then + call dump_columns( tape%File, tape%hlist, nflds, nocols, pverp, nfils, & + col_ndxs, chk_ndxs, phs_owners, phys_decomp ) + endif + if (has_dyn_ilev_flds) then + call dump_columns( tape%File, tape%hlist, nflds, nocols, pverp, nfils, & + fdyn_ndxs, ldyn_ndxs, dyn_owners, dyn_decomp ) + endif deallocate( col_ndxs, chk_ndxs, fdyn_ndxs, ldyn_ndxs, phs_owners, dyn_owners ) deallocate( mlons, mlats, phs_dists ) @@ -501,92 +522,167 @@ subroutine sat_hist_write( tape , nflds, nfils) end subroutine sat_hist_write !------------------------------------------------------------------------------- - subroutine dump_columns( File, hitem, ncols, nfils, fdims, ldims, owners ) - use cam_history_support, only: field_info, hentry, hist_coords, fillvalue - use pio, only: pio_initdecomp, pio_freedecomp, pio_setframe, pio_iam_iotask, pio_setdebuglevel, pio_offset_kind +! FIXME extra work > +! dump_columns routine is doing unnecessary extra work serially +! this happens because there is an unneeded mpi_allreduce call +! and then the gathered data is written in a serial manner; this +! could be improved by avoiding the mpi_allreduce call, and then +! writing local data out using pio_write_darray, which is parallel +! FIXME extra work < + subroutine dump_columns( File, hitems, nflds, ncols, nlevs, nfils, fdims, ldims, owners, decomp ) + use cam_history_support, only: field_info, hentry, fillvalue + use pio, only: pio_setframe, pio_offset_kind + use spmd_utils, only: mpi_real4, mpi_real8, mpicom, mpi_sum type(File_desc_t),intent(inout) :: File - type(hentry), intent(in), target :: hitem + type(hentry), intent(in), target :: hitems(:) + integer, intent(in) :: nflds integer, intent(in) :: ncols + integer, intent(in) :: nlevs integer, intent(in) :: nfils integer, intent(in) :: fdims(:) integer, intent(in) :: ldims(:) integer, intent(in) :: owners(:) + integer, intent(in) :: decomp + type(field_info), pointer :: field type(var_desc_t) :: vardesc - type(iosystem_desc_t), pointer :: sat_iosystem - type(io_desc_t) :: iodesc - integer :: t, ierr, ndims - integer, allocatable :: dimlens(:) + integer :: ierr - real(r8), allocatable :: buf(:) - integer, allocatable :: dof(:) - integer :: i,k, cnt + real(r8) :: sbuf1d(ncols),rbuf1d(ncols) + real(r4) :: buf1d(ncols) + real(r8) :: sbuf2d(nlevs,ncols), rbuf2d(nlevs,ncols) + real(r4) :: buf2d(nlevs,ncols) + integer :: i,k,f, cnt call t_startf ('sat_hist::dump_columns') - sat_iosystem => File%iosystem - field => hitem%field - vardesc = hitem%varid(1) - - - ndims=1 - if(associated(field%mdims)) then - ndims = size(field%mdims)+1 - else if(field%numlev>1) then - ndims=2 - end if - allocate(dimlens(ndims)) - dimlens(ndims)=ncols - if(ndims>2) then - do i=1,ndims-1 - dimlens(i)=hist_coords(field%mdims(i))%dimsize - enddo - else if(field%numlev>1) then - dimlens(1) = field%numlev - end if - - - allocate( buf( product(dimlens) ) ) - allocate( dof( product(dimlens) ) ) + do f = 1,nflds + field => hitems(f)%field + + if (field%numlev==nlevs .and. field%decomp_type==decomp) then + vardesc = hitems(f)%varid(1) + + if (nlevs==1) then + sbuf1d = 0.0_r8 + rbuf1d = 0.0_r8 + do i=1,ncols + if ( iam == owners(i) ) then + sbuf1d(i) = hitems(f)%hbuf( fdims(i), 1, ldims(i) ) + endif + enddo + ! FIXME extra work: unnecessary mpi call, then serial write + ! FIXME extra work: can use pio_write_darray on local data instead + call mpi_allreduce(sbuf1d,rbuf1d,ncols,mpi_real8, mpi_sum, mpicom, ierr) + buf1d(:) = real(rbuf1d(:),r4) + ierr = pio_put_var(File, vardesc, (/nfils/),(/ncols/), buf1d(:)) + if ( ierr /= PIO_NOERR ) then + call endrun('sat_hist::dump_columns: pio_put_var error') + endif + else + sbuf2d = 0.0_r8 + rbuf2d = 0.0_r8 + do i=1,ncols + if ( iam == owners(i) ) then + do k = 1,nlevs + sbuf2d(k,i) = hitems(f)%hbuf( fdims(i), k, ldims(i) ) + enddo + endif + enddo + ! FIXME extra work: unnecessary mpi call, then serial write + ! FIXME extra work: can use pio_write_darray on local data instead + call mpi_allreduce(sbuf2d,rbuf2d,ncols*nlevs,mpi_real8, mpi_sum, mpicom, ierr) + buf2d(:,:) = real(rbuf2d(:,:),r4) + ierr = pio_put_var(File, vardesc, (/1,nfils/),(/nlevs,ncols/), buf2d(:,:)) + if ( ierr /= PIO_NOERR ) then + call endrun('sat_hist::dump_columns: pio_put_var error') + endif + endif - cnt = 0 - buf = fillvalue - dof = 0 + endif - do i = 1,ncols - do k = 1,field%numlev - cnt = cnt+1 - if ( iam == owners(i) ) then - buf(cnt) = hitem%hbuf( fdims(i), k, ldims(i) ) - dof(cnt) = cnt - endif - enddo enddo - call pio_setframe(File, vardesc, int(-1,kind=PIO_OFFSET_KIND)) - - call pio_initdecomp(sat_iosystem, pio_double, dimlens, dof, iodesc ) + call t_stopf ('sat_hist::dump_columns') - call pio_setframe(File, vardesc, int(nfils,kind=PIO_OFFSET_KIND)) + end subroutine dump_columns - call pio_write_darray(File, vardesc, iodesc, buf, ierr, fillval=fillvalue) +!------------------------------------------------------------------------------- +! scan the fields for possible different decompositions +!------------------------------------------------------------------------------- + subroutine scan_flds( tape, nflds & + , has_phys_srf_flds, has_phys_lev_flds, has_phys_ilev_flds & + , has_dyn_srf_flds, has_dyn_lev_flds, has_dyn_ilev_flds ) + use cam_history_support, only : active_entry + use phys_grid, only: phys_decomp + use dyn_grid, only: dyn_decomp - call pio_freedecomp(sat_iosystem, iodesc) + type(active_entry), intent(in) :: tape + integer, intent(in) :: nflds + logical, save :: flds_scanned + logical, intent(out) :: has_phys_srf_flds + logical, intent(out) :: has_phys_lev_flds + logical, intent(out) :: has_phys_ilev_flds + logical, intent(out) :: has_dyn_srf_flds + logical, intent(out) :: has_dyn_lev_flds + logical, intent(out) :: has_dyn_ilev_flds + + integer :: f + character(len=cl) :: msg1, msg2 + + if (flds_scanned) return + + do f = 1,nflds + if ( tape%hlist(f)%field%decomp_type == phys_decomp ) then + if ( tape%hlist(f)%field%numlev == 1 ) then + has_phys_srf_flds = .true. + elseif ( tape%hlist(f)%field%numlev == pver ) then + has_phys_lev_flds = .true. + elseif ( tape%hlist(f)%field%numlev == pverp ) then + has_phys_ilev_flds = .true. + else + call endrun('sat_hist::scan_flds numlev error : '//tape%hlist(f)%field%name) + endif + elseif ( tape%hlist(f)%field%decomp_type == dyn_decomp ) then + if ( tape%hlist(f)%field%numlev == 1 ) then + has_dyn_srf_flds = .true. + elseif ( tape%hlist(f)%field%numlev == pver ) then + has_dyn_lev_flds = .true. + elseif ( tape%hlist(f)%field%numlev == pverp ) then + has_dyn_ilev_flds = .true. + else + call endrun('sat_hist::scan_flds numlev error : '//tape%hlist(f)%field%name) + endif + else + call endrun('sat_hist::scan_flds decomp_type error : '//tape%hlist(f)%field%name) + endif - deallocate( buf ) - deallocate( dof ) - deallocate( dimlens ) + ! Check that the only "mdim" is the vertical coordinate. + if (has_phys_srf_flds .or. has_phys_lev_flds .or. has_phys_ilev_flds .or. & + has_dyn_srf_flds .or. has_dyn_lev_flds .or. has_dyn_ilev_flds) then + ! The mdims pointer is unassociated on a restart. The restart initialization + ! should be fixed rather than requiring the check to make sure it is associated. + if (associated(tape%hlist(f)%field%mdims)) then + if (size(tape%hlist(f)%field%mdims) > 1) then + msg1 = 'sat_hist::scan_flds mdims error :'//tape%hlist(f)%field%name + msg2 = trim(msg1)//' has mdims in addition to the vertical coordinate.'//& + new_line('a')//' This is not currently supported.' + write(iulog,*) msg2 + call endrun(msg1) + end if + end if + end if - call t_stopf ('sat_hist::dump_columns') + enddo - end subroutine dump_columns + flds_scanned = .true. + end subroutine scan_flds !------------------------------------------------------------------------------- !------------------------------------------------------------------------------- subroutine read_next_position( ncols ) - use time_manager, only: get_curr_date, get_prev_date + use time_manager, only: get_curr_date use time_manager, only: set_time_float_from_date implicit none @@ -626,8 +722,14 @@ subroutine read_next_position( ncols ) call read_buffered_datetime( datetime, i ) - if ( datetime>begdatetime .and. beg_ndx<0 ) beg_ndx = i - if ( datetime>enddatetime ) exit bnds_loop + if (datetime > begdatetime .and. beg_ndx < 0) then + beg_ndx = i + end if + + if (datetime > enddatetime) then + exit bnds_loop + end if + end_ndx = i enddo bnds_loop @@ -660,7 +762,7 @@ end subroutine read_next_position !------------------------------------------------------------------------------- subroutine write_record_coord( tape, mod_lats, mod_lons, mod_dists, ncols, nfils ) - use time_manager, only: get_nstep, get_curr_date, get_curr_time + use time_manager, only: get_curr_date, get_curr_time use cam_history_support, only : active_entry implicit none type(active_entry), intent(inout) :: tape @@ -671,9 +773,8 @@ subroutine write_record_coord( tape, mod_lats, mod_lons, mod_dists, ncols, nfils real(r8), intent(in) :: mod_dists(ncols * sathist_nclosest) integer, intent(in) :: nfils - integer :: t, ierr, i + integer :: ierr, i integer :: yr, mon, day ! year, month, and day components of a date - integer :: nstep ! current timestep number integer :: ncdate ! current date in integer format [yyyymmdd] integer :: ncsec ! current time of day [seconds] integer :: ndcur ! day component of current time @@ -686,7 +787,6 @@ subroutine write_record_coord( tape, mod_lats, mod_lons, mod_dists, ncols, nfils call t_startf ('sat_hist::write_record_coord') - nstep = get_nstep() call get_curr_date(yr, mon, day, ncsec) ncdate = yr*10000 + mon*100 + day call get_curr_time(ndcur, nscur) diff --git a/components/eam/src/physics/cam/cam_diagnostics.F90 b/components/eam/src/physics/cam/cam_diagnostics.F90 index ead4f558f05b..5dca258775da 100644 --- a/components/eam/src/physics/cam/cam_diagnostics.F90 +++ b/components/eam/src/physics/cam/cam_diagnostics.F90 @@ -306,6 +306,7 @@ subroutine diag_init() call addfld ('MQ',(/ 'lev' /), 'A','kg/m2','Water vapor mass in layer') call addfld ('TMQ',horiz_only, 'A','kg/m2','Total (vertically integrated) precipitable water', & standard_name='atmosphere_mass_content_of_water_vapor') + call addfld ('TMQS',horiz_only, 'A','kg/m2','Total (vertically integrated) saturated precipitable water') call addfld ('TTQ',horiz_only, 'A', 'kg/m/s','Total (vertically integrated) vapor transport') call addfld ('TUQ',horiz_only, 'A','kg/m/s','Total (vertically integrated) zonal water flux') call addfld ('TVQ',horiz_only, 'A','kg/m/s','Total (vertically integrated) meridional water flux') @@ -1368,6 +1369,14 @@ subroutine diag_phys_writeout(state, psl) if (moist_physics) then + ! Mass of saturated q vertically integrated + call qsat(state%t(:ncol,:), state%pmid(:ncol,:), tem2(:ncol,:), ftem(:ncol,:)) + ftem(:ncol,:) = ftem(:ncol,:) * state%pdel(:ncol,:) * rga + do k=2,pver + ftem(:ncol,1) = ftem(:ncol,1) + ftem(:ncol,k) + end do + call outfld ('TMQS ',ftem, pcols ,lchnk ) + ! Relative humidity call qsat(state%t(:ncol,:), state%pmid(:ncol,:), & tem2(:ncol,:), ftem(:ncol,:)) diff --git a/components/eam/src/physics/cam/clubb_intr.F90 b/components/eam/src/physics/cam/clubb_intr.F90 index a93331fabdd0..33815759e7ca 100644 --- a/components/eam/src/physics/cam/clubb_intr.F90 +++ b/components/eam/src/physics/cam/clubb_intr.F90 @@ -20,7 +20,8 @@ module clubb_intr use shr_kind_mod, only: r8=>shr_kind_r8 use shr_log_mod , only: errMsg => shr_log_errMsg use ppgrid, only: pver, pverp - use phys_control, only: phys_getopts + use phys_control, only: phys_getopts, use_od_ss, use_od_fd + use od_common, only: od_ls_ncleff, od_bl_ncd, od_ss_sncleff use physconst, only: rair, cpair, gravit, latvap, latice, zvir, rh2o, karman, & tms_orocnst, tms_z0fac, pi use cam_logfile, only: iulog @@ -631,7 +632,8 @@ subroutine clubb_ini_cam(pbuf2d, dp1_in) use constituents, only: cnst_get_ind use phys_control, only: phys_getopts - use parameters_tunable, only: params_list + use parameters_tunable, only: params_list + use cam_abortutils, only: endrun #endif @@ -927,7 +929,25 @@ subroutine clubb_ini_cam(pbuf2d, dp1_in) call addfld ('VMAGDP', horiz_only, 'A', '-', 'ZM gustiness enhancement') call addfld ('VMAGCL', horiz_only, 'A', '-', 'CLUBB gustiness enhancement') call addfld ('TPERTBLT', horiz_only, 'A', 'K', 'perturbation temperature at PBL top') - + ! + if (use_od_fd) then + !added for turbulent orographic form drag (TOFD) output + call addfld ('DTAUX3_FD',(/'lev'/),'A','m/s2','U tendency - fd orographic drag') + call addfld ('DTAUY3_FD',(/'lev'/),'A','m/s2','V tendency - fd orographic drag') + call addfld ('DUSFC_FD',horiz_only,'A','N/m2','fd zonal oro surface stress') + call addfld ('DVSFC_FD',horiz_only,'A','N/m2','fd merio oro surface stress') + call add_default('DTAUX3_FD', 1, ' ') + call add_default('DTAUY3_FD', 1, ' ') + call add_default('DUSFC_FD', 1, ' ') + call add_default('DVSFC_FD', 1, ' ') + if (masterproc) then + write(iulog,*)'Using turbulent orographic form drag scheme (TOFD)' + end if + if (use_od_fd.and.do_tms) then + call endrun("clubb_intr: Both TMS and TOFD are turned on, please turn one off& + &by setting use_od_fd or do_tms as .false.") + end if + end if ! Initialize statistics, below are dummy variables dum1 = 300._r8 dum2 = 1200._r8 @@ -1155,7 +1175,9 @@ subroutine clubb_tend_cam( & use model_flags, only: ipdf_call_placement use advance_clubb_core_module, only: ipdf_post_advance_fields #endif - + use od_common, only: grid_size, oro_drag_interface + use hycoef, only: etamid + use physconst, only: rh2o,pi,rearth,r_universal implicit none ! --------------- ! @@ -1519,6 +1541,28 @@ subroutine clubb_tend_cam( & real(r8) :: sfc_v_diff_tau(pcols) ! Response to tau perturbation, m/s real(r8), parameter :: pert_tau = 0.1_r8 ! tau perturbation, Pa + !variables for turbulent orographic form drag (TOFD) interface + real(r8) :: dtaux3_fd(pcols,pver) + real(r8) :: dtauy3_fd(pcols,pver) + real(r8) :: dusfc_fd(pcols) + real(r8) :: dvsfc_fd(pcols) + logical :: gwd_ls,gwd_bl,gwd_ss,gwd_fd + real(r8) :: dummy_nm(pcols,pver) + real(r8) :: dummy_utgw(pcols,pver) + real(r8) :: dummy_vtgw(pcols,pver) + real(r8) :: dummy_ttgw(pcols,pver) + real(r8) :: dummx_ls(pcols,pver) + real(r8) :: dummx_bl(pcols,pver) + real(r8) :: dummx_ss(pcols,pver) + real(r8) :: dummy_ls(pcols,pver) + real(r8) :: dummy_bl(pcols,pver) + real(r8) :: dummy_ss(pcols,pver) + real(r8) :: dummx3_ls(pcols,pver) + real(r8) :: dummx3_bl(pcols,pver) + real(r8) :: dummx3_ss(pcols,pver) + real(r8) :: dummy3_ls(pcols,pver) + real(r8) :: dummy3_bl(pcols,pver) + real(r8) :: dummy3_ss(pcols,pver) real(r8) :: inv_exner_clubb_surf @@ -1947,6 +1991,35 @@ subroutine clubb_tend_cam( & call t_stopf('compute_tms') endif + if (use_od_fd) then + gwd_ls =.false. + gwd_bl =.false. + gwd_ss =.false. + gwd_fd =use_od_fd + dummy_nm =0.0_r8 + dummy_utgw=0.0_r8 + dummy_vtgw=0.0_r8 + dummy_ttgw=0.0_r8 + !sgh30 as the input for turbulent orographic form drag (TOFD) instead of sgh + call oro_drag_interface(state,cam_in,sgh30,pbuf,hdtime,dummy_nm,& + gwd_ls,gwd_bl,gwd_ss,gwd_fd,& + od_ls_ncleff,od_bl_ncd,od_ss_sncleff,& + dummy_utgw,dummy_vtgw,dummy_ttgw,& + dtaux3_ls=dummx3_ls,dtauy3_ls=dummy3_ls,& + dtaux3_bl=dummx3_bl,dtauy3_bl=dummy3_bl,& + dtaux3_ss=dummx3_ss,dtauy3_ss=dummy3_ss,& + dtaux3_fd=dtaux3_fd,dtauy3_fd=dtauy3_fd,& + dusfc_ls=dummx_ls,dvsfc_ls=dummy_ls,& + dusfc_bl=dummx_bl,dvsfc_bl=dummy_bl,& + dusfc_ss=dummx_ss,dvsfc_ss=dummy_ss,& + dusfc_fd=dusfc_fd,dvsfc_fd=dvsfc_fd) + + call outfld ('DTAUX3_FD', dtaux3_fd, pcols, lchnk) + call outfld ('DTAUY3_FD', dtauy3_fd, pcols, lchnk) + call outfld ('DUSFC_FD', dusfc_fd, pcols, lchnk) + call outfld ('DVSFC_FD', dvsfc_fd, pcols, lchnk) + endif + if (micro_do_icesupersat) then call physics_ptend_init(ptend_loc,state%psetcols, 'clubb_ice3', ls=.true., lu=.true., lv=.true., lq=lq) endif @@ -2067,7 +2140,14 @@ subroutine clubb_tend_cam( & dum_core_rknd = real((ksrftms(i)*state1%v(i,pver)), kind = core_rknd) vpwp_sfc = vpwp_sfc-(dum_core_rknd/rho_ds_zm(1)) endif - + ! ------------------------------------------------- ! + ! Apply TOFD + ! ------------------------------------------------- ! + ! tendency is flipped already + if (use_od_fd) then + um_forcing(2:pverp)=dtaux3_fd(i,pver:1:-1) + vm_forcing(2:pverp)=dtauy3_fd(i,pver:1:-1) + endif ! Need to flip arrays around for CLUBB core do k=1,pverp um_in(k) = real(um(i,pverp-k+1), kind = core_rknd) @@ -3091,7 +3171,7 @@ end subroutine clubb_tend_cam ! ! ! =============================================================================== ! - subroutine clubb_surface (state, cam_in, ustar, obklen) + subroutine clubb_surface (state, cam_in, pbuf, ustar, obklen) !------------------------------------------------------------------------------- ! Description: Provide the obukhov length and the surface friction velocity @@ -3108,10 +3188,12 @@ subroutine clubb_surface (state, cam_in, ustar, obklen) !------------------------------------------------------------------------------- use physics_types, only: physics_state - use physconst, only: zvir + use physconst, only: zvir,gravit use ppgrid, only: pver, pcols use constituents, only: cnst_get_ind use camsrfexch, only: cam_in_t + use hb_diff, only: pblintd_ri + use physics_buffer, only: pbuf_get_index, pbuf_get_field, physics_buffer_desc implicit none @@ -3119,8 +3201,9 @@ subroutine clubb_surface (state, cam_in, ustar, obklen) ! Input Auguments ! ! --------------- ! - type(physics_state), intent(inout) :: state ! Physics state variables - type(cam_in_t), intent(in) :: cam_in + type(physics_state), intent(inout) :: state ! Physics state variables + type(cam_in_t), intent(in) :: cam_in + type(physics_buffer_desc), pointer, intent(in) :: pbuf(:) ! ---------------- ! ! Output Auguments ! @@ -3136,16 +3219,23 @@ subroutine clubb_surface (state, cam_in, ustar, obklen) ! --------------- ! integer :: i ! indicees + integer :: k integer :: ncol ! # of atmospheric columns real(r8) :: th(pcols) ! surface potential temperature real(r8) :: thv(pcols) ! surface virtual potential temperature + real(r8) :: th_lv(pcols,pver) ! level potential temperature + real(r8) :: thv_lv(pcols,pver) ! level virtual potential temperature real(r8) :: kinheat ! kinematic surface heat flux real(r8) :: kinwat ! kinematic surface vapor flux real(r8) :: kbfs ! kinematic surface buoyancy flux + real(r8) :: kbfs_pcol(pcols) ! kinematic surface buoyancy flux stored for all pcols integer :: ixq,ixcldliq !PMA fix for thv real(r8) :: rrho ! Inverse air density + integer :: oro_drag_ribulk_idx ! pbuf index of bulk richardson number for oro drag + real(r8), pointer :: oro_drag_ribulk(:) ! pbuf pointer for bulk richardson number + #endif obklen(pcols) = 0.0_r8 @@ -3181,6 +3271,43 @@ subroutine clubb_surface (state, cam_in, ustar, obklen) kinheat, kinwat, kbfs, obklen(i) ) enddo + if (use_od_ss) then + !add calculation of bulk richardson number here + !compute the whole level th and thv for diagnose of bulk richardson number + thv_lv=0.0_r8 + th_lv =0.0_r8 + + !use the same virtual potential temperature formula as above (thv) except for all vertical levels + !used for bulk richardson number below in pblintd_ri + do i=1,ncol + do k=1,pver + th_lv(i,k) = state%t(i,k)*state%exner(i,k) + if (use_sgv) then + thv_lv(i,k) = th_lv(i,k)*(1.0_r8+zvir*state%q(i,k,ixq) & + - state%q(i,k,ixcldliq)) + else + thv_lv(i,k) = th_lv(i,k)*(1.0_r8+zvir*state%q(i,k,ixq)) + end if + enddo + enddo + + !recalculate the kbfs stored in kbfs_pcol for bulk richardson number in pblintd_ri + kbfs_pcol=0.0_r8 + do i=1,ncol + call calc_ustar( state%t(i,pver), state%pmid(i,pver), cam_in%wsx(i), cam_in%wsy(i), rrho, ustar(i) ) + call calc_obklen( th(i), thv(i), cam_in%cflx(i,1), cam_in%shf(i), rrho, ustar(i), & + kinheat, kinwat, kbfs, obklen(i) ) + kbfs_pcol(i)=kbfs + enddo + + oro_drag_ribulk_idx = pbuf_get_index('oro_drag_ribulk') + call pbuf_get_field(pbuf, oro_drag_ribulk_idx, oro_drag_ribulk) + + !calculate the bulk richardson number + call pblintd_ri(ncol, gravit, thv_lv, state%zm, state%u, state%v, & + ustar, obklen, kbfs_pcol, oro_drag_ribulk) + endif + return #endif diff --git a/components/eam/src/physics/cam/gw_convect.F90 b/components/eam/src/physics/cam/gw_convect.F90 index fc2fce99214a..dff87eec89c9 100644 --- a/components/eam/src/physics/cam/gw_convect.F90 +++ b/components/eam/src/physics/cam/gw_convect.F90 @@ -4,10 +4,10 @@ module gw_convect ! This module handles gravity waves from convection, and was extracted from ! gw_drag in May 2013. ! - -use gw_utils, only: r8 - -use gw_common, only: pver, pgwv +use cam_logfile, only: iulog +use spmd_utils, only: masterproc +use gw_utils, only: r8 +use gw_common, only: pver, pgwv implicit none private @@ -21,8 +21,8 @@ module gw_convect ! Dimension for mean wind in heating. integer :: maxuh -! Index for level at 700 mb. -integer :: k700 +! Index for level for storm/steering flow (usually 700 mb) +integer :: k_src_wind ! Table of source spectra. real(r8), allocatable :: mfcc(:,:,:) @@ -31,19 +31,21 @@ module gw_convect !========================================================================== -subroutine gw_convect_init(k700_in, mfcc_in, errstring) - ! Index at 700 mb. - integer, intent(in) :: k700_in - ! Source spectra to keep as table. - real(r8), intent(in) :: mfcc_in(:,:,:) - ! Report any errors from this routine. - character(len=*), intent(out) :: errstring - +subroutine gw_convect_init( plev_src_wind, mfcc_in, errstring) + use ref_pres, only: pref_edge + real(r8), intent(in) :: plev_src_wind ! reference pressure value [Pa] to set k_src_wind (previously hardcoded to 70000._r8) + real(r8), intent(in) :: mfcc_in(:,:,:) ! Source spectra to keep as table + character(len=*), intent(out) :: errstring ! Report any errors from this routine integer :: ierr + integer :: k errstring = "" - k700 = k700_in + do k = 0, pver + if ( pref_edge(k+1) < plev_src_wind ) k_src_wind = k+1 + end do + + if (masterproc) write (iulog,*) 'gw_convect: steering flow level = ',k_src_wind ! First dimension is maxh. maxh = size(mfcc_in,1) @@ -60,7 +62,9 @@ end subroutine gw_convect_init subroutine gw_beres_src(ncol, ngwv, lat, u, v, netdt, & zm, src_level, tend_level, tau, ubm, ubi, xv, yv, c, & - hdepth, maxq0, CF, hdepth_scaling_factor) + hdepth, maxq0_out, maxq0_conversion_factor, hdepth_scaling_factor, & + hdepth_min, storm_speed_min, & + use_gw_convect_old) !----------------------------------------------------------------------- ! Driver for multiple gravity wave drag parameterization. ! @@ -90,11 +94,20 @@ subroutine gw_beres_src(ncol, ngwv, lat, u, v, netdt, & real(r8), intent(in) :: zm(ncol,pver) ! Heating conversion factor - real(r8), intent(in) :: CF + real(r8), intent(in) :: maxq0_conversion_factor ! Scaling factor for the heating depth real(r8), intent(in) :: hdepth_scaling_factor + ! minimum hdepth for for spectrum lookup table + real(r8), intent(in) :: hdepth_min + + ! minimum convective storm speed + real(r8), intent(in) :: storm_speed_min + + ! switch for restoring legacy method + logical, intent(in) :: use_gw_convect_old + ! Indices of top gravity wave source level and lowest level where wind ! tendencies are allowed. integer, intent(out) :: src_level(ncol) @@ -110,19 +123,19 @@ subroutine gw_beres_src(ncol, ngwv, lat, u, v, netdt, & real(r8), intent(out) :: c(ncol,-pgwv:pgwv) ! Heating depth and maximum heating in each column. - real(r8), intent(out) :: hdepth(ncol), maxq0(ncol) + real(r8), intent(out) :: hdepth(ncol), maxq0_out(ncol) !---------------------------Local Storage------------------------------- ! Column and level indices. integer :: i, k - ! Zonal/meridional wind at 700mb. - real(r8) :: u700(ncol), v700(ncol) + ! Zonal/meridional source wind + real(r8) :: u_src(ncol), v_src(ncol) ! 3.14... real(r8), parameter :: pi = 4._r8*atan(1._r8) ! Maximum heating rate. - real(r8) :: q0(ncol) + real(r8) :: maxq0(ncol) ! Bottom/top heating range index. integer :: mini(ncol), maxi(ncol) @@ -135,36 +148,40 @@ subroutine gw_beres_src(ncol, ngwv, lat, u, v, netdt, & ! Source level tau for a column. real(r8) :: tau0(-PGWV:PGWV) ! Speed of convective cells relative to storm. - integer :: CS(ncol) + integer :: storm_speed(ncol) ! Index to shift spectra relative to ground. integer :: shift - ! Heating rate conversion factor. Change to take the value from caller and controllable by namelist (to tune QBO) - ! real(r8), parameter :: CF = 20._r8 - ! Averaging length. - real(r8), parameter :: AL = 1.0e5_r8 + ! fixed parameters (we may want to expose these in the namelist for tuning) + real(r8), parameter :: tau_avg_length = 100e3 ! spectrum averaging length [m] + real(r8), parameter :: heating_altitude_max = 20e3 ! max altitude [m] to check for max heating + + ! note: the heating_altitude_max is probably not needed because there is + ! rarely any convective heating above this level and the performance impact + ! of skipping the iteration over higher levels is likely negilible. + + integer :: ndepth_pos + integer :: ndepth_tot !---------------------------------------------------------------------- ! Initialize tau array !---------------------------------------------------------------------- - tau = 0.0_r8 + tau = 0.0_r8 hdepth = 0.0_r8 - q0 = 0.0_r8 - tau0 = 0.0_r8 + maxq0 = 0.0_r8 + tau0 = 0.0_r8 !------------------------------------------------------------------------ - ! Determine 700 mb layer wind and unit vectors, then project winds. + ! Determine source layer wind and unit vectors, then project winds. !------------------------------------------------------------------------ - ! Just use the 700 mb interface values for the source wind speed and - ! direction (unit vector). - - u700 = u(:,k700) - v700 = v(:,k700) + ! source wind speed and direction + u_src = u(:,k_src_wind) + v_src = v(:,k_src_wind) ! Get the unit vector components and magnitude at the surface. - call get_unit_vector(u700, v700, xv, yv, ubi(:,k700)) + call get_unit_vector(u_src, v_src, xv, yv, ubi(:,k_src_wind)) ! Project the local wind at midpoints onto the source wind. do k = 1, pver @@ -184,33 +201,62 @@ subroutine gw_beres_src(ncol, ngwv, lat, u, v, netdt, & ! which heating rate is continuously positive. !----------------------------------------------------------------------- - ! First find the indices for the top and bottom of the heating range. + ! Find indices for the top and bottom of the heating range. mini = 0 maxi = 0 - do k = pver, 1, -1 - do i = 1, ncol + + if (use_gw_convect_old) then + !--------------------------------------------------------------------- + ! original version used in CAM4/5/6 and EAMv1/2/3 + do k = pver, 1, -1 + do i = 1, ncol if (mini(i) == 0) then - ! Detect if we are outside the maximum range (where z = 20 km). - if (zm(i,k) >= 20000._r8) then - mini(i) = k - maxi(i) = k - else - ! First spot where heating rate is positive. - if (netdt(i,k) > 0.0_r8) mini(i) = k - end if + ! Detect if we are outside the maximum range (where z = 20 km). + if (zm(i,k) >= heating_altitude_max) then + mini(i) = k + maxi(i) = k + else + ! First spot where heating rate is positive. + if (netdt(i,k) > 0.0_r8) mini(i) = k + end if else if (maxi(i) == 0) then - ! Detect if we are outside the maximum range (z = 20 km). - if (zm(i,k) >= 20000._r8) then - maxi(i) = k - else - ! First spot where heating rate is no longer positive. - if (.not. (netdt(i,k) > 0.0_r8)) maxi(i) = k - end if + ! Detect if we are outside the maximum range (z = 20 km). + if (zm(i,k) >= heating_altitude_max) then + maxi(i) = k + else + ! First spot where heating rate is no longer positive. + if (.not. (netdt(i,k) > 0.0_r8)) maxi(i) = k + end if end if - end do - ! When all done, exit - if (all(maxi /= 0)) exit - end do + end do + ! When all done, exit + if (all(maxi /= 0)) exit + end do + !--------------------------------------------------------------------- + else + !--------------------------------------------------------------------- + ! cleaner version that addresses bug in original where heating max and + ! depth were too low whenever heating <=0 occurred in the middle of + ! the heating profile (ex. at the melting level) + do i = 1, ncol + do k = pver, 1, -1 + if ( zm(i,k) < heating_altitude_max ) then + if ( netdt(i,k) > 0.0_r8 ) then + ! Set mini as first spot where heating rate is positive + if ( mini(i)==0 ) mini(i) = k + ! Set maxi to current level + maxi(i) = k + end if + else + ! above the max check if indices were found + if ( mini(i)==0 ) mini(i) = k + if ( maxi(i)==0 ) maxi(i) = k + end if + end do + end do + !--------------------------------------------------------------------- + end if + ! Heating depth in km. hdepth = [ ( (zm(i,maxi(i))-zm(i,mini(i)))/1000._r8, i = 1, ncol ) ] @@ -223,19 +269,19 @@ subroutine gw_beres_src(ncol, ngwv, lat, u, v, netdt, & ! Maximum heating rate. do k = minval(maxi), maxval(mini) where (k >= maxi .and. k <= mini) - q0 = max(q0, netdt(:ncol,k)) + maxq0 = max(maxq0, netdt(:ncol,k)) end where end do !output max heating rate in K/day - maxq0 = q0*24._r8*3600._r8 + maxq0_out = maxq0*24._r8*3600._r8 ! Multipy by conversion factor - q0 = q0 * CF + maxq0 = maxq0 * maxq0_conversion_factor - ! Taking ubm at 700 mb to be the storm speed, find the cell speed where - ! the storm speed is > 10 m/s. - CS = int(sign(max(abs(ubm(:,k700))-10._r8, 0._r8), ubm(:,k700))) + ! Taking ubm at assumed source level to be the storm speed, + ! find the cell speed where the storm speed is > storm_speed_min + storm_speed = int(sign(max(abs(ubm(:,k_src_wind))-storm_speed_min, 0._r8), ubm(:,k_src_wind))) uh = 0._r8 do k = minval(maxi), maxval(mini) @@ -244,7 +290,7 @@ subroutine gw_beres_src(ncol, ngwv, lat, u, v, netdt, & end where end do - uh = uh - real(CS, r8) + uh = uh - real(storm_speed, r8) ! Limit uh to table range. uh = min(uh, real(maxuh, r8)) @@ -270,8 +316,19 @@ subroutine gw_beres_src(ncol, ngwv, lat, u, v, netdt, & !--------------------------------------------------------------------- ! Look up spectrum only if depth >= 2.5 km, else set tau0 = 0. !--------------------------------------------------------------------- - - if ((hdepth(i) >= 2.5_r8) .and. (abs(lat(i)) < (pi/2._r8))) then + if ((hdepth(i) >= hdepth_min)) then + ndepth_pos = 0 + ndepth_tot = 0 + do k = 1,pver + if ( k>=maxi(i).and.k<=mini(i) ) then + ndepth_tot = ndepth_tot + 1 + if ( netdt(i,k)>0 ) ndepth_pos = ndepth_pos + 1 + end if + end do + ! write (iulog,*) 'WHDEBUG - i: ',i,' Hd: ',hdepth(i),' Hn: ',ndepth_pos,' N: ',ndepth_tot + end if + + if ((hdepth(i) >= hdepth_min) .and. (abs(lat(i)) < (pi/2._r8))) then !------------------------------------------------------------------ ! Look up the spectrum using depth and uh. @@ -280,11 +337,11 @@ subroutine gw_beres_src(ncol, ngwv, lat, u, v, netdt, & tau0 = mfcc(NINT(hdepth(i)),NINT(uh(i)),:) ! Shift spectrum so that it is relative to the ground. - shift = -nint(real(CS(i), r8)/dc) + shift = -nint(storm_speed(i)/dc) tau0 = cshift(tau0,shift) ! Adjust magnitude. - tau0 = tau0*q0(i)*q0(i)/AL + tau0 = tau0*maxq0(i)*maxq0(i)/tau_avg_length ! Adjust for critical level filtering. Umini = max(nint(Umin(i)/dc),-PGWV) diff --git a/components/eam/src/physics/cam/gw_drag.F90 b/components/eam/src/physics/cam/gw_drag.F90 index 1c6b7920e85a..2f3c1652a68f 100644 --- a/components/eam/src/physics/cam/gw_drag.F90 +++ b/components/eam/src/physics/cam/gw_drag.F90 @@ -25,18 +25,21 @@ module gw_drag use shr_kind_mod, only: r8 => shr_kind_r8 use ppgrid, only: pcols, pver + use hycoef, only: hyai, hybi, hyam, hybm, etamid use constituents, only: pcnst use physics_types, only: physics_state, physics_ptend, physics_ptend_init use spmd_utils, only: masterproc use cam_history, only: outfld, hist_fld_active use cam_logfile, only: iulog - use cam_abortutils, only: endrun + use cam_abortutils,only: endrun use ref_pres, only: do_molec_diff, ntop_molec, nbot_molec - use physconst, only: cpair + use physconst, only: cpair, rh2o, zvir, pi, rearth, r_universal!zvir is the ep1 in wrf,rearth is the radius of earth(m),r_universal is the gas constant ! These are the actual switches for different gravity wave sources. - use phys_control, only: use_gw_oro, use_gw_front, use_gw_convect, use_gw_energy_fix + ! The orographic control switches are also here + use phys_control, only: use_gw_oro, use_gw_front, use_gw_convect, use_gw_energy_fix, use_od_ls, use_od_bl, use_od_ss + use od_common, only: od_ls_ncleff, od_bl_ncd, od_ss_sncleff ! Typical module header implicit none @@ -47,6 +50,7 @@ module gw_drag ! PUBLIC: interfaces ! public :: gw_drag_readnl ! Read namelist + public :: gw_register ! Register pbuf variables public :: gw_init ! Initialization public :: gw_tend ! interface to actual parameterization @@ -118,6 +122,11 @@ module gw_drag ! namelist logical :: history_amwg ! output the variables used by the AMWG diag package + logical :: use_gw_convect_old ! switch to enable legacy behavior + real(r8) :: gw_convect_plev_src_wind ! reference pressure level for source wind for convective GWD [Pa] + real(r8) :: gw_convect_hdepth_min ! minimum hdepth for for convective GWD spectrum lookup table [km] + real(r8) :: gw_convect_storm_speed_min ! minimum convective storm speed for convective GWD [m/s] + !========================================================================== contains !========================================================================== @@ -141,7 +150,9 @@ subroutine gw_drag_readnl(nlfile) namelist /gw_drag_nl/ pgwv, gw_dc, tau_0_ubc, effgw_beres, effgw_cm, & effgw_oro, fcrit2, frontgfc, gw_drag_file, taubgnd, gw_convect_hcf, & - hdepth_scaling_factor + hdepth_scaling_factor, gw_convect_hdepth_min, & + gw_convect_storm_speed_min, gw_convect_plev_src_wind, & + use_gw_convect_old !---------------------------------------------------------------------- if (masterproc) then @@ -170,8 +181,12 @@ subroutine gw_drag_readnl(nlfile) call mpibcast(frontgfc, 1, mpir8, 0, mpicom) call mpibcast(taubgnd, 1, mpir8, 0, mpicom) call mpibcast(gw_drag_file, len(gw_drag_file), mpichar, 0, mpicom) - call mpibcast(gw_convect_hcf, 1, mpir8, 0, mpicom) - call mpibcast(hdepth_scaling_factor, 1, mpir8, 0, mpicom) + call mpibcast(gw_convect_hcf, 1, mpir8, 0, mpicom) + call mpibcast(hdepth_scaling_factor, 1, mpir8, 0, mpicom) + call mpibcast(gw_convect_hdepth_min, 1, mpir8, 0, mpicom) + call mpibcast(gw_convect_storm_speed_min, 1, mpir8, 0, mpicom) + call mpibcast(gw_convect_plev_src_wind, 1, mpir8, 0, mpicom) + call mpibcast(use_gw_convect_old, 1, mpilog, 0, mpicom) #endif dc = gw_dc @@ -196,7 +211,16 @@ end subroutine gw_drag_readnl !========================================================================== -subroutine gw_init() +subroutine gw_register() + use od_common, only: oro_drag_register + + call oro_drag_register() + +end subroutine gw_register + +!========================================================================== + +subroutine gw_init(pbuf2d) !----------------------------------------------------------------------- ! Time independent initialization for multiple gravity wave ! parameterization. @@ -205,7 +229,7 @@ subroutine gw_init() use cam_history, only: addfld, horiz_only, add_default use interpolate_data, only: lininterp use phys_control, only: phys_getopts - use physics_buffer, only: pbuf_get_index + use physics_buffer, only: pbuf_get_index, physics_buffer_desc use ref_pres, only: pref_edge use physconst, only: gravit, rair @@ -215,6 +239,9 @@ subroutine gw_init() use gw_front, only: gw_front_init use gw_convect, only: gw_convect_init + use od_common, only: oro_drag_init + !------------------------------Arguments-------------------------------- + type(physics_buffer_desc), pointer :: pbuf2d(:,:) !---------------------------Local storage------------------------------- integer :: l, k @@ -224,7 +251,6 @@ subroutine gw_init() ! Index for levels at specific pressures. integer :: kfront - integer :: k700 ! output tendencies and state variables for CAM4 temperature, ! water vapor, cloud ice and cloud liquid budgets. @@ -288,6 +314,8 @@ subroutine gw_init() !----------------------------------------------------------------------- + call oro_drag_init(pbuf2d) + ! Set model flags. do_spectral_waves = (pgwv > 0 .and. (use_gw_front .or. use_gw_convect)) orographic_only = (use_gw_oro .and. .not. do_spectral_waves) @@ -362,13 +390,21 @@ subroutine gw_init() errstring) if (trim(errstring) /= "") call endrun("gw_common_init: "//errstring) - if (use_gw_oro) then - - if (effgw_oro == unset_r8) then + if (use_gw_oro.or.& + use_od_ls.or.& + use_od_bl.or.& + use_od_ss) then + ! + if (use_gw_oro.and.effgw_oro == unset_r8) then call endrun("gw_drag_init: Orographic gravity waves enabled, & &but effgw_oro was not set.") end if - + ! + if (use_gw_oro.and.use_od_ls) then + call endrun("gw_drag_init: Both orographic gravity waves schemes are turned on, & + &please turn one off by setting use_gw_oro or use_od_ls as .false.") + end if + ! call gw_oro_init(errstring) if (trim(errstring) /= "") call endrun("gw_oro_init: "//errstring) @@ -383,6 +419,36 @@ subroutine gw_init() 'Zonal gravity wave surface stress') call addfld ('TAUGWY',horiz_only, 'A','N/m2', & 'Meridional gravity wave surface stress') + if (use_od_ls.or.& + use_od_bl.or.& + use_od_ss) then + !added for orographic drag + call addfld ('DTAUX3_LS',(/'lev'/),'A','m/s2','U tendency - ls orographic drag') + call addfld ('DTAUY3_LS',(/'lev'/),'A','m/s2','V tendency - ls orographic drag') + call addfld ('DTAUX3_BL',(/'lev'/),'A','m/s2','U tendency - bl orographic drag') + call addfld ('DTAUY3_BL',(/'lev'/),'A','m/s2','V tendency - bl orographic drag') + call addfld ('DTAUX3_SS',(/'lev'/),'A','m/s2','U tendency - ss orographic drag') + call addfld ('DTAUY3_SS',(/'lev'/),'A','m/s2','V tendency - ss orographic drag') + call addfld ('DUSFC_LS',horiz_only,'A', 'N/m2', 'ls zonal oro surface stress') + call addfld ('DVSFC_LS',horiz_only,'A', 'N/m2', 'ls merio oro surface stress') + call addfld ('DUSFC_BL',horiz_only,'A', 'N/m2', 'bl zonal oro surface stress') + call addfld ('DVSFC_BL',horiz_only,'A', 'N/m2', 'bl merio oro surface stress') + call addfld ('DUSFC_SS',horiz_only,'A', 'N/m2', 'ss zonal oro surface stress') + call addfld ('DVSFC_SS',horiz_only,'A', 'N/m2', 'ss merio oro surface stress') + call add_default('DTAUX3_LS ', 1,' ') + call add_default('DTAUY3_LS ', 1,' ') + call add_default('DTAUX3_BL ', 1,' ') + call add_default('DTAUY3_BL ', 1,' ') + call add_default('DTAUX3_SS ', 1,' ') + call add_default('DTAUY3_SS ', 1,' ') + call add_default ('DUSFC_LS ', 1,' ') + call add_default ('DVSFC_LS ', 1,' ') + call add_default ('DUSFC_BL ', 1,' ') + call add_default ('DVSFC_BL ', 1,' ') + call add_default ('DUSFC_SS ', 1,' ') + call add_default ('DVSFC_SS ', 1,' ') + !added for orographic drag output + endif if (history_amwg) then call add_default('TAUGWX ', 1, ' ') @@ -451,20 +517,10 @@ subroutine gw_init() ttend_dp_idx = pbuf_get_index('TTEND_DP') - do k = 0, pver - ! 700 hPa index - if (pref_edge(k+1) < 70000._r8) k700 = k+1 - end do - - if (masterproc) then - write (iulog,*) 'K700 =',k700 - end if - ! Initialization of Beres' parameterization parameters call gw_init_beres(mfcc) - call gw_convect_init(k700, mfcc, errstring) - if (trim(errstring) /= "") & - call endrun("gw_convect_init: "//errstring) + call gw_convect_init(gw_convect_plev_src_wind, mfcc, errstring) + if (trim(errstring) /= "") call endrun("gw_convect_init: "//errstring) ! Output for gravity waves from the Beres scheme. call gw_spec_addflds(prefix=beres_pf, scheme="Beres", & @@ -583,12 +639,15 @@ subroutine gw_tend(state, sgh, pbuf, dt, ptend, cam_in) use camsrfexch, only: cam_in_t ! Location-dependent cpair use physconst, only: cpairv + use od_common, only: oro_drag_interface use gw_common, only: gw_prof, momentum_energy_conservation, & gw_drag_prof use gw_oro, only: gw_oro_src use gw_front, only: gw_cm_src use gw_convect, only: gw_beres_src use dycore, only: dycore_is + use phys_grid, only: get_rlat_all_p + use physconst, only: gravit,rair !------------------------------Arguments-------------------------------- type(physics_state), intent(in) :: state ! physics state structure ! Standard deviation of orography. @@ -598,6 +657,26 @@ subroutine gw_tend(state, sgh, pbuf, dt, ptend, cam_in) ! Parameterization net tendencies. type(physics_ptend), intent(out):: ptend type(cam_in_t), intent(in) :: cam_in + !locally added gw and bl drag + real(r8) :: dtaux3_ls(pcols,pver) + real(r8) :: dtauy3_ls(pcols,pver) + real(r8) :: dtaux3_bl(pcols,pver) + real(r8) :: dtauy3_bl(pcols,pver) + real(r8) :: dtaux3_ss(pcols,pver) + real(r8) :: dtauy3_ss(pcols,pver) + real(r8) :: dummx3_fd(pcols,pver) + real(r8) :: dummy3_fd(pcols,pver) + ! + real(r8) :: dusfc_ls(pcols) + real(r8) :: dvsfc_ls(pcols) + real(r8) :: dusfc_bl(pcols) + real(r8) :: dvsfc_bl(pcols) + real(r8) :: dusfc_ss(pcols) + real(r8) :: dvsfc_ss(pcols) + real(r8) :: dummx_fd(pcols) + real(r8) :: dummy_fd(pcols) + ! + real(r8) :: dx(pcols),dy(pcols) !---------------------------Local storage------------------------------- @@ -765,7 +844,9 @@ subroutine gw_tend(state, sgh, pbuf, dt, ptend, cam_in) ! Determine wave sources for Beres04 scheme call gw_beres_src(ncol, pgwv, state1%lat(:ncol), u, v, ttend_dp, & zm, src_level, tend_level, tau, ubm, ubi, xv, yv, c, & - hdepth, maxq0, gw_convect_hcf, hdepth_scaling_factor) + hdepth, maxq0, gw_convect_hcf, hdepth_scaling_factor, & + gw_convect_hdepth_min, gw_convect_storm_speed_min, & + use_gw_convect_old) do_latitude_taper = .false. @@ -879,7 +960,6 @@ subroutine gw_tend(state, sgh, pbuf, dt, ptend, cam_in) !--------------------------------------------------------------------- ! Orographic stationary gravity waves !--------------------------------------------------------------------- - ! Determine the orographic wave source call gw_oro_src(ncol, & u, v, t, sgh(:ncol), pmid, pint, dpm, zm, nm, & @@ -893,11 +973,35 @@ subroutine gw_tend(state, sgh, pbuf, dt, ptend, cam_in) piln, rhoi, nm, ni, ubm, ubi, xv, yv, & effgw_oro, c, kvtt, q, dse, tau, utgw, vtgw, & ttgw, qtgw, taucd, egwdffi, gwut(:,:,0:0), dttdf, dttke) - - ! Add the orographic tendencies to the spectrum tendencies - ! Compute the temperature tendency from energy conservation - ! (includes spectrum). - + endif + ! + if ( use_od_ls .or. use_od_bl .or. use_od_ss) then + utgw=0.0_r8 + vtgw=0.0_r8 + ttgw=0.0_r8 + call oro_drag_interface(state,cam_in,sgh,pbuf,dt,nm,& + use_od_ls,use_od_bl,use_od_ss,.false.,& + od_ls_ncleff,od_bl_ncd,od_ss_sncleff,& + utgw,vtgw,ttgw,& + dtaux3_ls=dtaux3_ls,dtauy3_ls=dtauy3_ls,& + dtaux3_bl=dtaux3_bl,dtauy3_bl=dtauy3_bl,& + dtaux3_ss=dtaux3_ss,dtauy3_ss=dtauy3_ss,& + dtaux3_fd=dummx3_fd,dtauy3_fd=dummy3_fd,& + dusfc_ls=dusfc_ls,dvsfc_ls=dvsfc_ls,& + dusfc_bl=dusfc_bl,dvsfc_bl=dvsfc_bl,& + dusfc_ss=dusfc_ss,dvsfc_ss=dvsfc_ss,& + dusfc_fd=dummx_fd,dvsfc_fd=dummy_fd) + endif + ! + ! Add the orographic tendencies to the spectrum tendencies + ! Compute the temperature tendency from energy conservation + ! (includes spectrum). + ! both old and new gwd scheme will add the tendency to circulation + ! + if (use_gw_oro.or.& + use_od_ls .or.& + use_od_bl .or.& + use_od_ss) then if(.not. use_gw_energy_fix) then !original do k = 1, pver @@ -906,11 +1010,11 @@ subroutine gw_tend(state, sgh, pbuf, dt, ptend, cam_in) vtgw(:,k) = vtgw(:,k) * cam_in%landfrac(:ncol) ptend%v(:ncol,k) = ptend%v(:ncol,k) + vtgw(:,k) ptend%s(:ncol,k) = ptend%s(:ncol,k) + ttgw(:,k) & - -(ptend%u(:ncol,k) * (u(:,k) + ptend%u(:ncol,k)*0.5_r8*dt) & - +ptend%v(:ncol,k) * (v(:,k) + ptend%v(:ncol,k)*0.5_r8*dt)) + -(ptend%u(:ncol,k) * (u(:,k) + ptend%u(:ncol,k)*0.5_r8*dt) & + +ptend%v(:ncol,k) * (v(:,k) + ptend%v(:ncol,k)*0.5_r8*dt)) ttgw(:,k) = ttgw(:,k) & - -(ptend%u(:ncol,k) * (u(:,k) + ptend%u(:ncol,k)*0.5_r8*dt) & - +ptend%v(:ncol,k) * (v(:,k) + ptend%v(:ncol,k)*0.5_r8*dt)) + -(ptend%u(:ncol,k) * (u(:,k) + ptend%u(:ncol,k)*0.5_r8*dt) & + +ptend%v(:ncol,k) * (v(:,k) + ptend%v(:ncol,k)*0.5_r8*dt)) ttgw(:,k) = ttgw(:,k) / cpairv(:ncol, k, lchnk) end do else @@ -947,12 +1051,34 @@ subroutine gw_tend(state, sgh, pbuf, dt, ptend, cam_in) call outfld('UTGWORO', utgw, ncol, lchnk) call outfld('VTGWORO', vtgw, ncol, lchnk) call outfld('TTGWORO', ttgw, ncol, lchnk) + ! + if (use_gw_oro) then + !old gwd scheme tau0x = tau(:,0,pver) * xv * effgw_oro tau0y = tau(:,0,pver) * yv * effgw_oro call outfld('TAUGWX', tau0x, ncol, lchnk) call outfld('TAUGWY', tau0y, ncol, lchnk) + endif + ! call outfld('SGH ', sgh,pcols, lchnk) - + ! + if (use_od_ls.or.& + use_od_bl.or.& + use_od_ss) then + call outfld ('DTAUX3_LS', dtaux3_ls, pcols, lchnk) + call outfld ('DTAUY3_LS', dtauy3_ls, pcols, lchnk) + call outfld ('DTAUX3_BL', dtaux3_bl, pcols, lchnk) + call outfld ('DTAUY3_BL', dtauy3_bl, pcols, lchnk) + call outfld ('DTAUX3_SS', dtaux3_ss, pcols, lchnk) + call outfld ('DTAUY3_SS', dtauy3_ss, pcols, lchnk) + call outfld ('DUSFC_LS', dusfc_ls, pcols, lchnk) + call outfld ('DVSFC_LS', dvsfc_ls, pcols, lchnk) + call outfld ('DUSFC_BL', dusfc_bl, pcols, lchnk) + call outfld ('DVSFC_BL', dvsfc_bl, pcols, lchnk) + call outfld ('DUSFC_SS', dusfc_ss, pcols, lchnk) + call outfld ('DVSFC_SS', dvsfc_ss, pcols, lchnk) + endif + ! end if ! Convert the tendencies for the dry constituents to dry air basis. diff --git a/components/eam/src/physics/cam/hb_diff.F90 b/components/eam/src/physics/cam/hb_diff.F90 index fdebeb1ee93a..3d18ce50280d 100644 --- a/components/eam/src/physics/cam/hb_diff.F90 +++ b/components/eam/src/physics/cam/hb_diff.F90 @@ -36,6 +36,8 @@ module hb_diff public init_hb_diff public compute_hb_diff public pblintd + !added for separation calculation of monin-obklen length + public pblintd_ri ! ! PBL limits ! @@ -764,5 +766,121 @@ subroutine austausch_pbl(lchnk ,ncol , & end do return end subroutine austausch_pbl + !=============================================================================== + subroutine pblintd_ri(ncol ,gravit , & + thv ,z ,u ,v , & + ustar ,obklen ,kbfs ,rino_bulk) + use pbl_utils, only: virtem, calc_ustar, calc_obklen + integer, intent(in) :: ncol ! number of atmospheric columns + real(r8), intent(in) :: gravit + real(r8), intent(in) :: thv(pcols,pver) ! virtual temperature + real(r8), intent(in) :: z(pcols,pver) ! height above surface [m] + real(r8), intent(in) :: u(pcols,pver) ! windspeed x-direction [m/s] + real(r8), intent(in) :: v(pcols,pver) ! windspeed y-direction [m/s] + real(r8), intent(in) :: ustar(pcols) ! surface friction velocity [m/s] + real(r8), intent(in) :: obklen(pcols) ! Obukhov length + real(r8), intent(in) :: kbfs(pcols) ! sfc kinematic buoyancy flux [m^2/s^3] + ! + ! Output arguments + ! + real(r8) :: wstar(pcols) ! convective sclae velocity [m/s] + real(r8) :: pblh(pcols) ! boundary-layer height [m] + real(r8) :: bge(pcols) ! buoyancy gradient enhancment + real(r8), intent(out) :: rino_bulk(pcols) ! bulk Richardson no. surface level + ! + !---------------------------Local parameters---------------------------- + ! + real(r8), parameter :: tiny = 1.e-36_r8 ! lower bound for wind magnitude + real(r8), parameter :: fac = 100._r8 ! ustar parameter in height diagnosis + ! + !---------------------------Local workspace----------------------------- + ! + integer :: i ! longitude index + integer :: k ! level index + real(r8) :: phiminv(pcols) ! inverse phi function for momentum + real(r8) :: phihinv(pcols) ! inverse phi function for heat + real(r8) :: rino(pcols,pver) ! bulk Richardson no. from level to ref lev + real(r8) :: tlv(pcols) ! ref. level pot tmp + tmp excess + real(r8) :: tref(pcols) ! ref. level pot tmp + real(r8) :: vvk ! velocity magnitude squared + logical :: unstbl(pcols) ! pts w/unstbl pbl (positive virtual ht flx) + logical :: check(pcols) ! True=>chk if Richardson no.>critcal + ! + do i=1,ncol + check(i) = .true. + rino(i,pver) = 0.0_r8 + rino_bulk(i) = 0.0_r8 + pblh(i) = z(i,pver) + tref(i) = thv(i,pver)!if not excess then tref is equal to lowest level thv_lv + end do + ! + ! PBL height calculation: Scan upward until the Richardson number between + ! the first level and the current level exceeds the "critical" value. + ! + do k=pver-1,pver-npbl+1,-1 + do i=1,ncol + if (check(i)) then + vvk = (u(i,k) - u(i,pver))**2 + (v(i,k) - v(i,pver))**2 + fac*ustar(i)**2 + vvk = max(vvk,tiny) + rino(i,k) = gravit*(thv(i,k) - thv(i,pver))*(z(i,k)-z(i,pver))/(thv(i,pver)*vvk) + if (rino(i,k) >= ricr) then + pblh(i) = z(i,k+1) + (ricr - rino(i,k+1))/(rino(i,k) - rino(i,k+1)) * & + (z(i,k) - z(i,k+1)) + check(i) = .false. + end if + end if + end do + end do + ! + ! Estimate an effective surface temperature to account for surface fluctuations + ! + do i=1,ncol + if (check(i)) pblh(i) = z(i,pverp-npbl) + unstbl(i) = (kbfs(i) > 0._r8) + check(i) = (kbfs(i) > 0._r8) + if (check(i)) then + phiminv(i) = (1._r8 - binm*pblh(i)/obklen(i))**onet + rino(i,pver) = 0.0_r8 + tlv(i) = thv(i,pver) + kbfs(i)*fak/( ustar(i)*phiminv(i) ) + tref(i) = tlv(i) + end if + end do + ! + ! Improve pblh estimate for unstable conditions using the convective temperature excess: + ! + do i = 1,ncol + bge(i) = 1.e-8_r8 + end do + do k=pver-1,pver-npbl+1,-1 + do i=1,ncol + if (check(i)) then + vvk = (u(i,k) - u(i,pver))**2 + (v(i,k) - v(i,pver))**2 + fac*ustar(i)**2 + vvk = max(vvk,tiny) + rino(i,k) = gravit*(thv(i,k) - tlv(i))*(z(i,k)-z(i,pver))/(thv(i,pver)*vvk) + if (rino(i,k) >= ricr) then + pblh(i) = z(i,k+1) + (ricr - rino(i,k+1))/(rino(i,k) - rino(i,k+1))* & + (z(i,k) - z(i,k+1)) + bge(i) = 2._r8*gravit/(thv(i,k)+thv(i,k+1))*(thv(i,k)-thv(i,k+1))/(z(i,k)-z(i,k+1))*pblh(i) + if (bge(i).lt.0._r8) then + bge(i) = 1.e-8_r8 + endif + check(i) = .false. + end if + end if + end do + end do + ! + !calculate bulk richardson number in the surface layer + !following Holstag and Boville (1993) equation (2.8) + ! + do i=1,ncol + vvk = u(i,pver)**2 + v(i,pver)**2 + fac*ustar(i)**2 + vvk = max(vvk,tiny) + rino_bulk(i)=gravit*(thv(i,pver) - tref(i))*z(i,pver)/(thv(i,pver)*vvk) + enddo + ! + return + end subroutine pblintd_ri + !=============================================================================== end module hb_diff diff --git a/components/eam/src/physics/cam/od_common.F90 b/components/eam/src/physics/cam/od_common.F90 new file mode 100644 index 000000000000..52658d2dbecb --- /dev/null +++ b/components/eam/src/physics/cam/od_common.F90 @@ -0,0 +1,1693 @@ +module od_common +!========================================================================== +! This module contains code common to different orographic drag +! parameterizations. +! It includes 4 parts: +! orographic gravity wave drag (Xie et al.,2020), +! flow-blocking drag (Xie et al.,2020), +! small-scale orographic gravity wave drag (Tsiringakis et al. 2017), +! turbulent orographic form drag (Beljaars et al.,2004). +!========================================================================== +use shr_kind_mod, only: i8 => shr_kind_i8, r8 => shr_kind_r8 +use shr_sys_mod, only: shr_sys_flush +use ppgrid, only: pcols, pver, begchunk, endchunk +use cam_logfile, only: iulog +use cam_abortutils,only: endrun +use spmd_utils, only: masterproc +use pio, only: file_desc_t +use phys_control, only: use_od_ls, use_od_bl, use_od_ss +use physics_buffer,only: dtype_r8, physics_buffer_desc, pbuf_get_chunk +use physics_buffer,only: pbuf_get_index, pbuf_get_field, pbuf_add_field, pbuf_set_field + +implicit none +private +save + +! Public interface. +public :: oro_drag_readnl +public :: oro_drag_register +public :: oro_drag_init +public :: oro_drag_interface +public :: od_gsd,pblh_get_level_idx,grid_size + +type(file_desc_t), pointer :: topo_file_ncid + +! dimensions for topo shape data +integer, parameter :: ndir_asymmetry = 2+1 ! add 1 to avoid bug reading file - not sure why this happens +integer, parameter :: ndir_efflength = 180 ! 1-degree resolution with opposite directions mirrored + +! pbuf indices for data read in from topo data file +integer :: oro_drag_convexity_idx = -1 ! Convexity +integer :: oro_drag_asymmetry_idx = -1 ! Asymmetry +integer :: oro_drag_efflength_idx = -1 ! Effective length +integer :: oro_drag_ribulk_idx = -1 ! bulk richardson number (calculated in CLUBB) + +!tunable parameter to the od schemes +real(r8),public, protected :: od_ls_ncleff = 3._r8 !tunable parameter for oGWD +real(r8),public, protected :: od_bl_ncd = 3._r8 !tunable parameter for FBD +real(r8),public, protected :: od_ss_sncleff= 1._r8 !tunable parameter for sGWD + +contains + +!========================================================================== + +subroutine oro_drag_readnl(nlfile) + + use namelist_utils, only: find_group_name + use units, only: getunit, freeunit + use mpishorthand + + ! File containing namelist input. + character(len=*), intent(in) :: nlfile + + ! Local variables + integer :: unitn, ierr + character(len=*), parameter :: subname = 'oro_drag_readnl' + + ! More specific name for dc to prevent a name clash or confusion in the + ! namelist. + + namelist /oro_drag_nl/ od_ls_ncleff, od_bl_ncd, od_ss_sncleff + !--------------------------------------------------------------------- + !read oro_drag_nl only when use the od schemes + if (use_od_ls.or.use_od_bl.or.use_od_ss) then + if (masterproc) then + unitn = getunit() + open( unitn, file=trim(nlfile), status='old' ) + call find_group_name(unitn, 'oro_drag_nl', status=ierr) + if (ierr == 0) then + read(unitn, oro_drag_nl, iostat=ierr) + if (ierr /= 0) then + call endrun(subname // ':: ERROR reading namelist') + end if + end if + close(unitn) + call freeunit(unitn) + end if + + if (masterproc) write(iulog,*) "oro_drag_readnl od_ls_ncleff, od_bl_ncd, od_ss_sncleff ",od_ls_ncleff,od_bl_ncd,od_ss_sncleff + +#ifdef SPMD + ! Broadcast namelist variables + call mpibcast(od_ls_ncleff, 1, mpir8, 0, mpicom) + call mpibcast(od_bl_ncd, 1, mpir8, 0, mpicom) + call mpibcast(od_ss_sncleff, 1, mpir8, 0, mpicom) +#endif + ! + endif + +end subroutine oro_drag_readnl + +!========================================================================== + +subroutine oro_drag_open_topo_file() + use filenames, only: bnd_topo + use ioFileMod, only: getfil + use cam_pio_utils,only: cam_pio_openfile + use pio, only: pio_nowrite + include 'netcdf.inc' + !----------------------------------------------------------------------- + character(len=256) :: bnd_topo_loc ! filepath of topo file on local disk + allocate(topo_file_ncid) + call getfil(bnd_topo, bnd_topo_loc) + call cam_pio_openfile(topo_file_ncid, bnd_topo_loc, PIO_NOWRITE) +end subroutine oro_drag_open_topo_file + +!========================================================================== + +subroutine oro_drag_close_topo_file + use pio, only: pio_closefile + call pio_closefile(topo_file_ncid) + deallocate(topo_file_ncid) + nullify(topo_file_ncid) +end subroutine oro_drag_close_topo_file + +!========================================================================== + +subroutine oro_drag_register() + !----------------------------------------------------------------------- + ! Register pbuf variables for orographic drag parameterizations + !----------------------------------------------------------------------- + ! create pbuf variables to hold oro drag data + if (use_od_ls.or.use_od_bl) then + call pbuf_add_field('oro_drag_convexity','physpkg',dtype_r8,(/pcols/), oro_drag_convexity_idx) + call pbuf_add_field('oro_drag_asymmetry','physpkg',dtype_r8,(/pcols,ndir_asymmetry/),oro_drag_asymmetry_idx) + call pbuf_add_field('oro_drag_efflength','physpkg',dtype_r8,(/pcols,ndir_efflength/),oro_drag_efflength_idx) + end if + if (use_od_ss) then + call pbuf_add_field('oro_drag_ribulk', 'physpkg',dtype_r8,(/pcols/), oro_drag_ribulk_idx) + end if + +end subroutine oro_drag_register + +!========================================================================== + +subroutine oro_drag_init(pbuf2d) + !----------------------------------------------------------------------- + ! Initialization for orographic drag parameterizations + !----------------------------------------------------------------------- + use pio, only: file_desc_t + use ncdio_atm, only: infld + use cam_grid_support, only: cam_grid_check, cam_grid_get_decomp, cam_grid_id,cam_grid_get_dim_names + use infnan, only: nan, assignment(=) + !----------------------------------------------------------------------- + type(physics_buffer_desc), pointer :: pbuf2d(:,:) + !----------------------------------------------------------------------- + logical :: found + character(len=8) :: dim1name, dim2name + character*11 :: subname='oro_drag_init' + integer :: grid_id + integer :: c + + real(r8), allocatable:: oro_drag_convexity_tmp(:,:) + real(r8), allocatable:: oro_drag_asymmetry_tmp(:,:,:) + real(r8), allocatable:: oro_drag_efflength_tmp(:,:,:) + + type(physics_buffer_desc), pointer :: pbuf_chunk(:) ! temporary pbuf pointer for single chunk + !----------------------------------------------------------------------- + if (.not.(use_od_ls.or.use_od_bl)) return + + grid_id = cam_grid_id('physgrid') + if (.not. cam_grid_check(grid_id)) then + call endrun(trim(subname)//': Internal error, no "physgrid" grid') + end if + + ! Alocate variables for reading oro drag data + allocate( oro_drag_convexity_tmp(pcols,begchunk:endchunk) ) + allocate( oro_drag_asymmetry_tmp(pcols,ndir_asymmetry,begchunk:endchunk) ) + allocate( oro_drag_efflength_tmp(pcols,ndir_efflength,begchunk:endchunk) ) + oro_drag_convexity_tmp(:,:) = nan + oro_drag_asymmetry_tmp(:,:,:) = nan + oro_drag_efflength_tmp(:,:,:) = nan + + ! Read special orographic shape fields from topo file + call cam_grid_get_dim_names(grid_id, dim1name, dim2name) + call oro_drag_open_topo_file() + + found=.false. + call infld( 'OC', topo_file_ncid, dim1name, dim2name, 1, pcols, & + begchunk, endchunk, oro_drag_convexity_tmp(:,:), found, gridname='physgrid') + if(.not. found) call endrun('ERROR - oro_drag_init: topo file read error - OC') + + found=.false. + call infld( 'OA', topo_file_ncid, dim1name, 'ndir_asymmetry', dim2name, 1, pcols, 1, ndir_asymmetry, & + begchunk, endchunk, oro_drag_asymmetry_tmp(:,:,:), found, gridname='physgrid') + if(.not. found) call endrun('ERROR - oro_drag_init: topo file read error - OA') + + found=.false. + call infld( 'OL', topo_file_ncid, dim1name, 'ndir_efflength', dim2name, 1, pcols, 1, ndir_efflength, & + begchunk, endchunk, oro_drag_efflength_tmp(:,:,:), found, gridname='physgrid') + if(.not. found) call endrun('ERROR - oro_drag_init: topo file read error - OL') + + call oro_drag_close_topo_file() + + ! copy the oro drag data in pbuf + do c=begchunk,endchunk + pbuf_chunk => pbuf_get_chunk(pbuf2d, c) + call pbuf_set_field(pbuf_chunk, oro_drag_convexity_idx, oro_drag_convexity_tmp(:,c) ) + call pbuf_set_field(pbuf_chunk, oro_drag_asymmetry_idx, oro_drag_asymmetry_tmp(:,:,c) ) + call pbuf_set_field(pbuf_chunk, oro_drag_efflength_idx, oro_drag_efflength_tmp(:,:,c) ) + end do + + deallocate(oro_drag_convexity_tmp) + deallocate(oro_drag_asymmetry_tmp) + deallocate(oro_drag_efflength_tmp) + +end subroutine oro_drag_init +!========================================================================== + +subroutine oro_drag_interface(state, cam_in, sgh, pbuf, dtime, nm, & + gwd_ls, gwd_bl, gwd_ss, gwd_fd, & + od_ls_ncleff, od_bl_ncd,od_ss_sncleff, & + utgw, vtgw, ttgw, & + dtaux3_ls,dtauy3_ls,dtaux3_bl,dtauy3_bl, & + dtaux3_ss,dtauy3_ss,dtaux3_fd,dtauy3_fd, & + dusfc_ls, dvsfc_ls ,dusfc_bl, dvsfc_bl, & + dusfc_ss, dvsfc_ss ,dusfc_fd, dvsfc_fd) + use physics_types, only: physics_state + use camsrfexch, only: cam_in_t + use ppgrid, only: pcols,pver,pverp + use physconst, only: gravit,rair,cpair,rh2o,zvir,pi + use hycoef, only: etamid + !----------------------------------------------------------------------- + type(physics_state), intent(in) :: state ! physics state structure + type(cam_in_t), intent(in) :: cam_in + real(r8), intent(in) :: sgh(pcols) + type(physics_buffer_desc), pointer :: pbuf(:) ! Physics buffer + real(r8), intent(in) :: dtime + real(r8), intent(in) :: nm(state%ncol,pver) ! midpoint Brunt-Vaisalla frequency + !options for the 4 schemes + logical , intent(in) :: gwd_ls + logical , intent(in) :: gwd_bl + logical , intent(in) :: gwd_ss + logical , intent(in) :: gwd_fd + !tunable parameter from namelist + real(r8), intent(in) :: od_ls_ncleff + real(r8), intent(in) :: od_bl_ncd + real(r8), intent(in) :: od_ss_sncleff + !vertical profile of the momentum tendencies + real(r8), intent(out), optional :: utgw(state%ncol,pver) + real(r8), intent(out), optional :: vtgw(state%ncol,pver) + real(r8), intent(out), optional :: ttgw(state%ncol,pver) + !output drag terms in 3D and surface + real(r8), intent(out), optional :: dtaux3_ls(pcols,pver) + real(r8), intent(out), optional :: dtauy3_ls(pcols,pver) + real(r8), intent(out), optional :: dtaux3_bl(pcols,pver) + real(r8), intent(out), optional :: dtauy3_bl(pcols,pver) + real(r8), intent(out), optional :: dtaux3_ss(pcols,pver) + real(r8), intent(out), optional :: dtauy3_ss(pcols,pver) + real(r8), intent(out), optional :: dtaux3_fd(pcols,pver) + real(r8), intent(out), optional :: dtauy3_fd(pcols,pver) + real(r8), intent(out), optional :: dusfc_ls(pcols) + real(r8), intent(out), optional :: dvsfc_ls(pcols) + real(r8), intent(out), optional :: dusfc_bl(pcols) + real(r8), intent(out), optional :: dvsfc_bl(pcols) + real(r8), intent(out), optional :: dusfc_ss(pcols) + real(r8), intent(out), optional :: dvsfc_ss(pcols) + real(r8), intent(out), optional :: dusfc_fd(pcols) + real(r8), intent(out), optional :: dvsfc_fd(pcols) + + real(r8) :: ztop(pcols,pver) ! top interface height asl (m) + real(r8) :: zbot(pcols,pver) ! bottom interface height asl (m) + real(r8) :: zmid(pcols,pver) ! middle interface height asl (m) + real(r8) :: dz(pcols,pver) ! model layer height + + integer :: pblh_idx = 0 + integer :: kpbl2d_in(pcols) + integer :: kpbl2d_reverse_in(pcols) + real(r8), pointer :: pblh(:) + real(r8) :: dx(pcols),dy(pcols) + + real(r8), pointer :: oro_drag_convexity(:) + real(r8), pointer :: oro_drag_asymmetry(:,:) + real(r8), pointer :: oro_drag_efflength(:,:) + real(r8), pointer :: oro_drag_ribulk(:) ! pbuf pointer for bulk richardson number + + integer :: ncol + integer :: i + integer :: k + !----------------------------------------------------------------------- + + ncol=state%ncol + !convert heights above surface to heights above sea level + !obtain z,dz,dx,dy,and k for pblh + kpbl2d_in=0_r8 + kpbl2d_reverse_in=0_r8 + ztop=0._r8 + zbot=0._r8 + zmid=0._r8 + dusfc_ls=0._r8 + dvsfc_ls=0._r8 + dusfc_bl=0._r8 + dvsfc_bl=0._r8 + dusfc_ss=0._r8 + dvsfc_ss=0._r8 + dusfc_fd=0._r8 + dvsfc_fd=0._r8 + dtaux3_ls=0._r8 + dtaux3_bl=0._r8 + dtauy3_ls=0._r8 + dtauy3_bl=0._r8 + dtaux3_ss=0._r8 + dtaux3_fd=0._r8 + dtauy3_ss=0._r8 + dtauy3_fd=0._r8 + + do k=1,pver + do i=1,ncol + ! assign values for level top/bottom + ztop(i,k)=state%zi(i,k) + zbot(i,k)=state%zi(i,k+1) + enddo + end do + + !transform adding the pressure + !transfer from surface to sea level + do k=1,pver + do i=1,ncol + ztop(i,k)=ztop(i,k)+state%phis(i)/gravit + zbot(i,k)=zbot(i,k)+state%phis(i)/gravit + zmid(i,k)=state%zm(i,k)+state%phis(i)/gravit + !dz is from bottom to top already for gw_drag + dz(i,k)=ztop(i,k)-zbot(i,k) + end do + end do + !get the layer index of pblh in layer for input in drag scheme + pblh_idx = pbuf_get_index('pblh') + call pbuf_get_field(pbuf, pblh_idx, pblh) + do i=1,pcols + kpbl2d_in(i)=pblh_get_level_idx(zbot(i,:)-(state%phis(i)/gravit),pblh(i)) + kpbl2d_reverse_in(i)=pverp-kpbl2d_in(i)!pverp-k + end do + + call pbuf_get_field(pbuf, oro_drag_convexity_idx, oro_drag_convexity ) + call pbuf_get_field(pbuf, oro_drag_asymmetry_idx, oro_drag_asymmetry ) + call pbuf_get_field(pbuf, oro_drag_efflength_idx, oro_drag_efflength ) + call pbuf_get_field(pbuf, oro_drag_ribulk_idx, oro_drag_ribulk) + + !get grid size for dx,dy + call grid_size(state,dx,dy) + + !interface for orographic drag + call od_gsd(u3d=state%u(:ncol,pver:1:-1),v3d=state%v(:ncol,pver:1:-1),t3d=state%t(:ncol,pver:1:-1),& + qv3d=state%q(:ncol,pver:1:-1,1),p3d=state%pmid(:ncol,pver:1:-1),p3di=state%pint(:ncol,pver+1:1:-1),& + pi3d=state%exner(:ncol,pver:1:-1),z=zbot(:ncol,pver:1:-1),& + od_ls_ncleff=od_ls_ncleff,od_bl_ncd=od_bl_ncd,od_ss_sncleff=od_ss_sncleff,& + rublten=utgw(:ncol,pver:1:-1),rvblten=vtgw(:ncol,pver:1:-1),rthblten=ttgw(:ncol,pver:1:-1),& + dtaux3d_ls=dtaux3_ls(:ncol,pver:1:-1),dtauy3d_ls=dtauy3_ls(:ncol,pver:1:-1),& + dtaux3d_bl=dtaux3_bl(:ncol,pver:1:-1),dtauy3d_bl=dtauy3_bl(:ncol,pver:1:-1),& + dtaux3d_ss=dtaux3_ss(:ncol,pver:1:-1),dtauy3d_ss=dtauy3_ss(:ncol,pver:1:-1),& + dtaux3d_fd=dtaux3_fd(:ncol,pver:1:-1),dtauy3d_fd=dtauy3_fd(:ncol,pver:1:-1),& + dusfcg_ls=dusfc_ls(:ncol),dvsfcg_ls=dvsfc_ls(:ncol),& + dusfcg_bl=dusfc_bl(:ncol),dvsfcg_bl=dvsfc_bl(:ncol),& + dusfcg_ss=dusfc_ss(:ncol),dvsfcg_ss=dvsfc_ss(:ncol),& + dusfcg_fd=dusfc_fd(:ncol),dvsfcg_fd=dvsfc_fd(:ncol),& + xland=cam_in%landfrac,br=oro_drag_ribulk(:ncol),& + var2d=sgh(:ncol),& + oc12d=oro_drag_convexity(:ncol),& + oa2d=oro_drag_asymmetry(:ncol,:),& + ol2d=oro_drag_efflength(:ncol,:),& + znu=etamid(pver:1:-1),dz=dz(:ncol,pver:1:-1),pblh=pblh(:ncol),& + cp=cpair,g=gravit,rd=rair,rv=rh2o,ep1=zvir,pi=pi,bnvbg=nm(:ncol,pver:1:-1),& + dt=dtime,dx=dx,dy=dy,& + kpbl2d=kpbl2d_reverse_in,gwd_opt=0,& + ids=1,ide=ncol,jds=0,jde=0,kds=1,kde=pver, & + ims=1,ime=ncol,jms=0,jme=0,kms=1,kme=pver, & + its=1,ite=ncol,jts=0,jte=0,kts=1,kte=pver, & + gwd_ls=gwd_ls,gwd_bl=gwd_bl,gwd_ss=gwd_ss,gwd_fd=gwd_fd ) + +end subroutine oro_drag_interface + +!========================================================================== + +function pblh_get_level_idx(height_array,pblheight) + implicit none + real(r8),intent(in),dimension(pver) :: height_array + real(r8),intent(in) :: pblheight + integer :: pblh_get_level_idx + !local + integer :: k + logical :: found + + pblh_get_level_idx = -1 + found=.false. + !get the pblh level index and return + do k = 1, pver + if((pblheight >= height_array(k+1).and.pblheight 300._r8) then + kpbl2 = k + IF (k == kpbl(i)) then + hpbl2 = hpbl(i)+10._r8 + ELSE + hpbl2 = za(i,k)+10._r8 + ENDIF + exit + ENDIF + enddo + + if(xland1(i).gt.0._r8 .and. 2._r8*var(i).le.hpbl(i))then + if(br1(i).gt.0._r8 .and. thvx(i,kpbl2)-thvx(i,kts) > 0._r8)then + cleff = sqrt(dxy(i)**2_r8 + dxyp(i)**2_r8) + cleff = (2.0_r8/sncleff) * max(dxmax_ss,cleff) + coefm(i) = (1._r8 + ol(i)) ** (oa1(i)+1._r8) + xlinv(i) = coefm(i) / cleff + govrth(i)=g/(0.5_r8*(thvx(i,kpbl2)+thvx(i,kts))) + bnrf=sqrt(govrth(i)*(thvx(i,kpbl2)-thvx(i,kts))/hpbl2) + + if(abs(bnrf/u1(i,kpbl2)).gt.xlinv(i))then + tauwavex0=0.5_r8*bnrf*xlinv(i)*(2._r8*MIN(var(i),varmax))**2_r8*ro(i,kvar)*u1(i,kvar) + tauwavex0=tauwavex0*ss_taper ! "Scale-awareness" + else + tauwavex0=0._r8 + endif + + if(abs(bnrf/v1(i,kpbl2)).gt.xlinv(i))then + tauwavey0=0.5_r8*bnrf*xlinv(i)*(2._r8*MIN(var(i),varmax))**2._r8*ro(i,kvar)*v1(i,kvar) + tauwavey0=tauwavey0*ss_taper ! "Scale-awareness" + else + tauwavey0=0._r8 + endif + + do k=kts,kpbl(i) !MIN(kpbl2+1,kte-1) + utendwave(i,k)=-1._r8*tauwavex0*2._r8*max((1._r8-za(i,k)/hpbl2),0._r8)/hpbl2 + vtendwave(i,k)=-1._r8*tauwavey0*2._r8*max((1._r8-za(i,k)/hpbl2),0._r8)/hpbl2 + enddo + endif + endif + enddo ! end i loop + + do k = kts,kte + do i = its,ite + dudt(i,k) = dudt(i,k) + utendwave(i,k) + dvdt(i,k) = dvdt(i,k) + vtendwave(i,k) + dtaux2d_ss(i,k) = utendwave(i,k) + dtauy2d_ss(i,k) = vtendwave(i,k) + dusfc_ss(i) = dusfc_ss(i) + utendwave(i,k) * del(i,k) + dvsfc_ss(i) = dvsfc_ss(i) + vtendwave(i,k) * del(i,k) + enddo + enddo + + ENDIF ! end if gsd_gwd_ss == .true. + !================================================================ + !add Beljaars et al. (2004, QJRMS, equ. 16) form drag: + !================================================================ + IF (gsd_gwd_fd.and.(ss_taper.GT.1.E-02) ) THEN + + utendform=0._r8 + vtendform=0._r8 + zq=0._r8 + + if (.not.gsd_gwd_ss.and.(ss_taper.GT.1.E-02) ) THEN + ! Defining layer height. This is already done above is small-scale GWD is used + do k = kts,kte + do i = its,ite + zq(i,k+1) = dz2(i,k)+zq(i,k) + enddo + enddo + + do k = kts,kte + do i = its,ite + za(i,k) = 0.5_r8*(zq(i,k)+zq(i,k+1)) + enddo + enddo + endif + + do i=its,ite + if (xland1(i) .gt. 0..and.2._r8*var(i).gt.0) then + ! refer to Beljaars (2004) eq.16. + a1=0.00026615161_r8*var(i)**2_r8 + a2=a1*0.005363_r8 + do k=kts,kte + wsp=SQRT(u1(i,k)**2_r8 + v1(i,k)**2_r8) + ! refer to Beljaars (2004) eq.16. + ! alpha*beta*Cmd*Ccorr*2.109 = 12.*1.*0.005*0.6*2.109 = 0.0759 + utendform(i,k)=-0.0759_r8*wsp*u1(i,k)* & + EXP(-(za(i,k)/1500._r8)**1.5_r8)*a2*za(i,k)**(-1.2_r8)*ss_taper + vtendform(i,k)=-0.0759_r8*wsp*v1(i,k)* & + EXP(-(za(i,k)/1500._r8)**1.5_r8)*a2*za(i,k)**(-1.2_r8)*ss_taper + enddo + endif + enddo + + do k = kts,kte + do i = its,ite + dudt(i,k) = dudt(i,k) + utendform(i,k) + dvdt(i,k) = dvdt(i,k) + vtendform(i,k) + !limit drag tendency + !some tendency is likely to even overturn the wind, + !making wind reverse in 1 timestep and reverse again in next, + !this limitation may help to make model stable, + !and no more wind reversal due to drag, + !which is suppose to decelerate, not accelerate + utendform(i,k) = sign(min(abs(utendform(i,k)),abs(u1(i,k))/deltim),utendform(i,k)) + vtendform(i,k) = sign(min(abs(vtendform(i,k)),abs(v1(i,k))/deltim),vtendform(i,k)) + dtaux2d_fd(i,k) = utendform(i,k) + dtauy2d_fd(i,k) = vtendform(i,k) + dusfc_fd(i) = dusfc_fd(i) + utendform(i,k) * del(i,k) + dvsfc_fd(i) = dvsfc_fd(i) + vtendform(i,k) * del(i,k) + enddo + enddo + ENDIF ! end if gsd_gwd_fd == .true. + !======================================================= + ! More for the large-scale gwd component + !======================================================= + IF (gsd_gwd_ls.and.(ls_taper.GT.1.E-02) ) THEN + ! + ! now compute vertical structure of the stress. + ! + do k = kts,kpblmax + do i = its,ite + if (k .le. kbl(i)) taup(i,k) = taub(i) + enddo + enddo + + if (scorer_on) then + ! + !determination of the interface height for scorer adjustment + ! + do i=its,ite + iint=.false. + do k=kpblmin,kte-1 + if (k.gt.kbl(i).and.usqj(i,k)-usqj(i,k-1).lt.0.and.(.not.iint)) then + iint=.true. + zl_hint(i)=zl(i,k+1) + endif + enddo + enddo + endif + + do k = kpblmin, kte-1 ! vertical level k loop! + kp1 = k + 1 + do i = its,ite + ! + ! unstablelayer if ri < ric + ! unstable layer if upper air vel comp along surf vel <=0 (crit lay) + ! at (u-c)=0. crit layer exists and bit vector should be set (.le.) + ! + if (k .ge. kbl(i)) then + !we modify the criteria for unstable layer + !that the lv is critical under 0.25 + !while we keep wave breaking ric for + !other larger lv + icrilv(i) = icrilv(i) .or. ( usqj(i,k) .lt. ric_rig)& + .or. (velco(i,k) .le. 0.0_r8) + brvf(i) = max(bnv2(i,k),bnv2min) ! brunt-vaisala frequency squared + brvf(i) = sqrt(brvf(i)) ! brunt-vaisala frequency + endif + enddo + + do i = its,ite + if (k .ge. kbl(i) .and. (.not. ldrag(i))) then + if (.not.icrilv(i) .and. taup(i,k) .gt. 0.0_r8 ) then + temv = 1.0_r8 / velco(i,k) + tem1 = coefm(i)/(dxy(i)/ncleff)*(ro(i,kp1)+ro(i,k))*brvf(i)*velco(i,k)*0.5_r8 + hd = sqrt(taup(i,k) / tem1) + fro = brvf(i) * hd * temv + ! + ! rim is the minimum-richardson number by shutts (1985) + ! + tem2 = sqrt(usqj(i,k)) + tem = 1._r8 + tem2 * fro + rim = usqj(i,k) * (1._r8-fro) / (tem * tem) + + ! + ! check stability to employ the 'saturation hypothesis' + ! of lindzen (1981) except at tropospheric downstream regions + ! + if (rim .le. ric) then ! saturation hypothesis! + if ((oa1(i) .le. 0._r8).or.(kp1 .ge. kpblmin )) then + temc = 2.0_r8 + 1.0_r8 / tem2 + hd = velco(i,k) * (2.0_r8*sqrt(temc)-temc) / brvf(i) + taup(i,kp1) = tem1 * hd * hd + ! + ! taup is restricted to monotoncally decrease + ! to avoid unexpected high taup in calculation + ! + taup(i,kp1)=min(tem1*hd*hd,taup(i,k)) + ! + ! add vertical decrease at low level below hint (Kim and Doyle 2005) + ! where Ri first decreases + ! + if (scorer_on.and.k.gt.klowtop(i).and.zl(i,k).le.zl_hint(i).and.k.lt.kte-1) then + l1=(9.81_r8*bnv2(i,kp1)/velco(i,kp1)**2) + l2=(9.81_r8*bnv2(i,k)/velco(i,k)**2) + taup(i,kp1)=min(taup(i,k),taup(i,k)*(l1/l2),tem1*hd*hd) + endif + endif + else ! no wavebreaking! + taup(i,kp1) = taup(i,k) + endif + endif + endif + enddo + enddo + + if(lcap.lt.kte) then + do klcap = lcapp1,kte + do i = its,ite + taup(i,klcap) = prsi(i,klcap) / prsi(i,lcap) * taup(i,lcap) + enddo + enddo + endif + + ENDIF !END LARGE-SCALE TAU CALCULATION + !=============================================================== + !COMPUTE BLOCKING COMPONENT + !=============================================================== + IF (gsd_gwd_bl.and.(ls_taper .GT. 1.E-02)) THEN + + do i = its,ite + if(.not.ldrag(i)) then + ! + !------- determine the height of flow-blocking layer + ! + kblk = 0 + pe = 0.0_r8 + + do k = kte, kpblmin, -1 + if(kblk.eq.0 .and. k.le.komax(i)) then + !flow block appears within the reference level + !compare potential energy and kinetic energy + !divided by g*ro is to turn del(pa) into height + pe = pe + bnv2(i,k)*(zl(i,komax(i))-zl(i,k))*del(i,k)/g/ro(i,k) + ke = 0.5_r8*((rcs*u1(i,k))**2._r8+(rcs*v1(i,k))**2._r8) + ! + !---------- apply flow-blocking drag when pe >= ke + ! + if(pe.ge.ke) then + kblk = k + kblk = min(kblk,kbl(i)) + zblk = zl(i,kblk)-zl(i,kts) + endif + endif + enddo + + if(kblk.ne.0) then + ! + !--------- compute flow-blocking stress + ! + + !dxmax_ls is different than the usual one + !because the taper is very different + !dxy is a length scale mostly in the direction of the flow to the ridge + !so it is good and not needed for an uneven grid area + !ref Lott and Miller (1997) original scheme + cd = max(2.0_r8-1.0_r8/od(i),0.0_r8) + ! + !tuning of the drag magnitude + cd=ncd*cd + ! + taufb(i,kts) = 0.5_r8 * roll(i) * coefm(i) / max(dxmax_ls,dxy(i))**2 * cd * dxyp(i) & + * olp(i) * zblk * ulow(i)**2 + !changed grid box area into dy*dy + tautem = taufb(i,kts)/float(kblk-kts) + do k = kts+1, kblk + taufb(i,k) = taufb(i,k-1) - tautem + enddo + + ! + !----------sum orographic GW stress and flow-blocking stress + ! + !taup(i,:) = taup(i,:) + taufb(i,:) ! Keep taup and taufb separate for now + endif + endif + enddo + + ENDIF ! end blocking drag +!=========================================================== + IF (gsd_gwd_ls.OR.gsd_gwd_bl.and.(ls_taper .GT. 1.E-02)) THEN + ! + ! calculate - (g)*d(tau)/d(pressure) and deceleration terms dtaux, dtauy + ! + do k = kts,kte + do i = its,ite + taud_ls(i,k) = 1._r8 * (taup(i,k+1) - taup(i,k)) * csg / del(i,k) + taud_bl(i,k) = 1._r8 * (taufb(i,k+1) - taufb(i,k)) * csg / del(i,k) + enddo + enddo + ! + ! limit de-acceleration (momentum deposition ) at top to 1/2 value + ! the idea is some stuff must go out the 'top' + ! + do klcap = lcap,kte + do i = its,ite + taud_ls(i,klcap) = taud_ls(i,klcap) * factop + taud_bl(i,klcap) = taud_bl(i,klcap) * factop + enddo + enddo + ! + ! if the gravity wave drag would force a critical line + ! in the lower ksmm1 layers during the next deltim timestep, + ! then only apply drag until that critical line is reached. + ! + do k = kts,kpblmax-1 + do i = its,ite + if (k .le. kbl(i)) then + if((taud_ls(i,k)+taud_bl(i,k)).ne.0._r8) & + dtfac(i) = min(dtfac(i),abs(velco(i,k) & + /(deltim*rcs*(taud_ls(i,k)+taud_bl(i,k))))) + endif + enddo + enddo + + do k = kts,kte + do i = its,ite + taud_ls(i,k) = taud_ls(i,k) * dtfac(i) * ls_taper + !apply limiter for ogwd + !1.dudt < |c-u|/dt, so u-c cannot change sign(u^n+1 = u^n + du/dt * dt) + !2.dudtchk if Richardson no.>critcal -#ifdef SCREAM_CONFIG_IS_CMAKE - if (use_cxx) then - call pblintd_f(& - shcol,nlev,nlevi,npbl,& ! Input - z,zi,thl,ql,& ! Input - q,u,v,& ! Input - ustar,obklen,kbfs,cldn,& ! Input - pblh) ! Output - return - endif -#endif - ! ! Compute Obukhov length virtual temperature flux and various arrays for use later: ! @@ -4343,9 +3927,6 @@ subroutine pblintd_init_pot(& shcol,nlev,& ! Input thl,ql,q,& ! Input thv) ! Output -#ifdef SCREAM_CONFIG_IS_CMAKE - use shoc_iso_f, only: shoc_pblintd_init_pot_f -#endif !------------------------------Arguments-------------------------------- ! ! Input arguments @@ -4364,13 +3945,6 @@ subroutine pblintd_init_pot(& integer :: k ! level index real(rtype) :: th -#ifdef SCREAM_CONFIG_IS_CMAKE - if (use_cxx) then - call shoc_pblintd_init_pot_f(shcol,nlev,thl,ql,q,& ! Input - thv) ! Output - return - endif -#endif ! Compute virtual potential temperature do k=1,nlev do i=1,shcol @@ -4418,10 +3992,6 @@ subroutine pblintd_height(& thv,thv_ref,& ! Input pblh,rino,check) ! Output -#ifdef SCREAM_CONFIG_IS_CMAKE - use shoc_iso_f, only: pblintd_height_f -#endif - !------------------------------Arguments-------------------------------- ! ! Input arguments @@ -4450,14 +4020,6 @@ subroutine pblintd_height(& integer :: k ! level index real(rtype) :: vvk ! velocity magnitude squared -#ifdef SCREAM_CONFIG_IS_CMAKE - if (use_cxx) then - call pblintd_height_f(shcol,nlev,npbl,z,u,v,ustar,thv,thv_ref,& ! Input - pblh,rino,check) ! Output - return - endif -#endif - ! ! PBL height calculation: Scan upward until the Richardson number between ! the first level and the current level exceeds the "critical" value. @@ -4485,10 +4047,6 @@ subroutine pblintd_surf_temp(& tlv,& ! Output pblh,check,rino) ! InOutput -#ifdef SCREAM_CONFIG_IS_CMAKE - use shoc_iso_f, only: pblintd_surf_temp_f -#endif - !------------------------------Arguments-------------------------------- ! Input arguments ! @@ -4519,15 +4077,6 @@ subroutine pblintd_surf_temp(& real(rtype), parameter :: sffrac= 0.1_rtype ! Surface layer fraction of boundary layer real(rtype), parameter :: binm = betam*sffrac ! betam * sffrac -#ifdef SCREAM_CONFIG_IS_CMAKE - if (use_cxx) then - call pblintd_surf_temp_f(shcol,nlev,nlevi,& ! Input - z,ustar,obklen,kbfs,thv,& ! Input - tlv,pblh,check,rino) ! InOutput - return - endif -#endif - ! ! Estimate an effective surface temperature to account for surface ! fluctuations @@ -4550,10 +4099,6 @@ subroutine pblintd_check_pblh(& z,ustar,check,& ! Input pblh) ! Output -#ifdef SCREAM_CONFIG_IS_CMAKE - use shoc_iso_f, only: pblintd_check_pblh_f -#endif - !------------------------------Arguments-------------------------------- ! Input arguments ! @@ -4573,13 +4118,6 @@ subroutine pblintd_check_pblh(& ! integer :: i ! longitude index -#ifdef SCREAM_CONFIG_IS_CMAKE - if (use_cxx) then - call pblintd_check_pblh_f(shcol,nlev,nlevi,npbl,z,ustar,check,pblh) - return - endif -#endif - ! ! PBL height must be greater than some minimum mechanical mixing depth ! Several investigators have proposed minimum mechanical mixing depth @@ -4606,10 +4144,6 @@ subroutine pblintd_cldcheck( & zi,cldn, & ! Input pblh) ! InOutput -#ifdef SCREAM_CONFIG_IS_CMAKE - use shoc_iso_f, only: shoc_pblintd_cldcheck_f -#endif - !------------------------------Arguments-------------------------------- ! Input arguments ! @@ -4629,13 +4163,6 @@ subroutine pblintd_cldcheck( & integer :: i ! longitude index logical(btype) :: cldcheck(shcol) ! True=>if cloud in lowest layer -#ifdef SCREAM_CONFIG_IS_CMAKE - if (use_cxx) then - call shoc_pblintd_cldcheck_f(shcol, nlev, nlevi, zi, cldn, pblh) - return - endif -#endif - ! ! Final requirement on PBL heightis that it must be greater than the depth ! of the lowest model level if there is any cloud diagnosed in @@ -4660,10 +4187,6 @@ end subroutine pblintd_cldcheck subroutine linear_interp(x1,x2,y1,y2,km1,km2,ncol,minthresh) -#ifdef SCREAM_CONFIG_IS_CMAKE - use shoc_iso_f, only: linear_interp_f -#endif - implicit none integer, intent(in) :: km1, km2 @@ -4675,13 +4198,6 @@ subroutine linear_interp(x1,x2,y1,y2,km1,km2,ncol,minthresh) integer :: k1, k2, i -#ifdef SCREAM_CONFIG_IS_CMAKE - if (use_cxx) then - call linear_interp_f(x1,x2,y1,y2,km1,km2,ncol,minthresh) - return - endif -#endif - #if 1 !i = check_grid(x1,x2,km1,km2,ncol) if (km1 .eq. km2+1) then @@ -4749,10 +4265,6 @@ subroutine compute_brunt_shoc_length(nlev,nlevi,shcol,dz_zt,thv,thv_zi,brunt) ! ! Computes the brunt_visala frequency -#ifdef SCREAM_CONFIG_IS_CMAKE - use shoc_iso_f, only: compute_brunt_shoc_length_f -#endif - implicit none integer, intent(in) :: nlev, nlevi, shcol ! Grid difference centereted on thermo grid [m] @@ -4765,13 +4277,6 @@ subroutine compute_brunt_shoc_length(nlev,nlevi,shcol,dz_zt,thv,thv_zi,brunt) real(rtype), intent(out) :: brunt(shcol, nlev) integer k, i -#ifdef SCREAM_CONFIG_IS_CMAKE - if (use_cxx) then - call compute_brunt_shoc_length_f(nlev,nlevi,shcol,dz_zt,thv,thv_zi,brunt) - return - endif -#endif - do k=1,nlev do i=1,shcol brunt(i,k) = (ggr/thv(i,k)) * (thv_zi(i,k) - thv_zi(i,k+1))/dz_zt(i,k) @@ -4785,10 +4290,6 @@ subroutine compute_l_inf_shoc_length(nlev,shcol,zt_grid,dz_zt,tke,l_inf) !========================================================= ! -#ifdef SCREAM_CONFIG_IS_CMAKE - use shoc_iso_f, only: compute_l_inf_shoc_length_f -#endif - implicit none integer, intent(in) :: nlev, shcol real(rtype), intent(in) :: zt_grid(shcol,nlev), dz_zt(shcol,nlev), tke(shcol,nlev) @@ -4796,13 +4297,6 @@ subroutine compute_l_inf_shoc_length(nlev,shcol,zt_grid,dz_zt,tke,l_inf) real(rtype) :: tkes, numer(shcol), denom(shcol) integer k, i -#ifdef SCREAM_CONFIG_IS_CMAKE - if (use_cxx) then - call compute_l_inf_shoc_length_f(nlev,shcol,zt_grid,dz_zt,tke,l_inf) - return - endif -#endif - numer(:) = 0._rtype denom(:) = 0._rtype @@ -4822,10 +4316,6 @@ end subroutine compute_l_inf_shoc_length subroutine compute_shoc_mix_shoc_length(nlev,shcol,tke,brunt,zt_grid,l_inf,shoc_mix) -#ifdef SCREAM_CONFIG_IS_CMAKE - use shoc_iso_f, only: compute_shoc_mix_shoc_length_f -#endif - implicit none integer, intent(in) :: nlev, shcol @@ -4848,14 +4338,6 @@ subroutine compute_shoc_mix_shoc_length(nlev,shcol,tke,brunt,zt_grid,l_inf,shoc_ ! Turnover timescale [s] real(rtype), parameter :: tscale = 400._rtype -#ifdef SCREAM_CONFIG_IS_CMAKE - if (use_cxx) then - call compute_shoc_mix_shoc_length_f(nlev,shcol,tke,brunt,zt_grid,l_inf,& !Input - shoc_mix) ! Ouptut - return - endif -#endif - brunt2(:,:) = 0.0 do k=1,nlev @@ -4876,10 +4358,6 @@ subroutine check_length_scale_shoc_length(nlev,shcol,host_dx,host_dy,shoc_mix) ! Do checks on the length scale. Make sure it is not ! larger than the grid mesh of the host model. -#ifdef SCREAM_CONFIG_IS_CMAKE - use shoc_iso_f, only: check_length_scale_shoc_length_f -#endif - implicit none integer, intent(in) :: nlev, shcol real(rtype), intent(in) :: host_dx(shcol), host_dy(shcol) @@ -4887,13 +4365,6 @@ subroutine check_length_scale_shoc_length(nlev,shcol,host_dx,host_dy,shoc_mix) real(rtype), intent(inout) :: shoc_mix(shcol, nlev) integer k, i -#ifdef SCREAM_CONFIG_IS_CMAKE - if (use_cxx) then - call check_length_scale_shoc_length_f(nlev,shcol,host_dx,host_dy,shoc_mix) - return - endif -#endif - do k=1,nlev do i=1,shcol shoc_mix(i,k)=min(maxlen,shoc_mix(i,k)) diff --git a/components/eam/src/physics/clubb/advance_windm_edsclrm_module.F90 b/components/eam/src/physics/clubb/advance_windm_edsclrm_module.F90 index 72d2e4d214bd..970de0b4d7fe 100644 --- a/components/eam/src/physics/clubb/advance_windm_edsclrm_module.F90 +++ b/components/eam/src/physics/clubb/advance_windm_edsclrm_module.F90 @@ -3,6 +3,9 @@ !=============================================================================== module advance_windm_edsclrm_module + !options for turbulent orographic form drag in wind forcing + use phys_control, only: use_od_fd + implicit none private ! Set Default Scope @@ -1571,8 +1574,12 @@ subroutine compute_uv_tndcy( solve_type, fcor, perp_wind_m, perp_wind_g, xm_forc endif else ! implemented in a host model. - - xm_tndcy = 0.0_core_rknd + !use forcing when using turbulent orographic form drag (TOFD) forcing + if (use_od_fd) then + xm_tndcy(1:gr%nz) = xm_forcing(1:gr%nz) + else + xm_tndcy = 0.0_core_rknd + endif endif diff --git a/components/eam/src/physics/crm/dummy_modules/clubb_intr.F90 b/components/eam/src/physics/crm/dummy_modules/clubb_intr.F90 new file mode 100644 index 000000000000..d6729e33f6b3 --- /dev/null +++ b/components/eam/src/physics/crm/dummy_modules/clubb_intr.F90 @@ -0,0 +1,31 @@ +module clubb_intr +!------------------------------------------------------------------------------- +! Dummy module to override src/physics/cam/clubb_intr.F90 +!------------------------------------------------------------------------------- +use shr_kind_mod, only: r8=>shr_kind_r8 +public :: clubb_implements_cnst +public :: clubb_init_cnst +public :: clubb_readnl +contains +!=============================================================================== +function clubb_implements_cnst(name) + ! Return true if specified constituent is implemented + character(len=*), intent(in) :: name ! constituent name + logical :: clubb_implements_cnst ! return value + clubb_implements_cnst = .false. +end function clubb_implements_cnst +!=============================================================================== +subroutine clubb_init_cnst(name, q, gcid) + ! Initialize the state if clubb_do_adv + character(len=*), intent(in) :: name ! constituent name + real(r8), intent(out) :: q(:,:) ! mass mixing ratio (gcol, plev) + integer, intent(in) :: gcid(:) ! global column id + return +end subroutine clubb_init_cnst +!=============================================================================== +subroutine clubb_readnl(nlfile) + character(len=*), intent(in) :: nlfile ! filepath for file containing namelist input + return +end subroutine clubb_readnl +!=============================================================================== +end module clubb_intr diff --git a/components/eam/src/physics/crm/pam/CMakeLists.txt b/components/eam/src/physics/crm/pam/CMakeLists.txt index 4a8f88f669b0..ac629fa89fde 100644 --- a/components/eam/src/physics/crm/pam/CMakeLists.txt +++ b/components/eam/src/physics/crm/pam/CMakeLists.txt @@ -9,8 +9,9 @@ set(PAM_DRIVER_SRC params.F90) add_library(pam_driver - ${PAM_DRIVER_SRC}) + ${PAM_DRIVER_SRC}) +set(SCREAM_LIBS_ONLY TRUE) set(SCREAM_HOME ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../..) add_library(eamxx_physics INTERFACE ${SCREAM_HOME}/components/eamxx/src/physics/) diff --git a/components/eam/src/physics/crm/pam/external b/components/eam/src/physics/crm/pam/external index 3ea20ad38f28..6b162a8892e7 160000 --- a/components/eam/src/physics/crm/pam/external +++ b/components/eam/src/physics/crm/pam/external @@ -1 +1 @@ -Subproject commit 3ea20ad38f286730973429e0d491420a6f599f11 +Subproject commit 6b162a8892e788d0a672d519374a3c82a0de2a5b diff --git a/components/eam/src/physics/crm/pam/pam_driver.cpp b/components/eam/src/physics/crm/pam/pam_driver.cpp index c7038343f08a..e3070cce240a 100644 --- a/components/eam/src/physics/crm/pam/pam_driver.cpp +++ b/components/eam/src/physics/crm/pam/pam_driver.cpp @@ -20,7 +20,6 @@ // Needed for p3_init #include "p3_functions.hpp" -#include "p3_f90.hpp" #include "pam_debug.h" bool constexpr enable_check_state = false; @@ -203,7 +202,8 @@ extern "C" void pam_driver() { #if defined(P3_CXX) if (is_first_step || is_restart) { auto am_i_root = coupler.get_option("am_i_root"); - scream::p3::p3_init(/*write_tables=*/false, am_i_root); + using P3F = scream::p3::Functions; + P3F::p3_init(/*write_tables=*/false, am_i_root); pam::p3_init_lookup_tables(); // Load P3 lookup table data - avoid re-loading every CRM call } #endif diff --git a/components/eam/src/physics/crm/pam/pam_statistics.h b/components/eam/src/physics/crm/pam/pam_statistics.h index bb277600dfa1..8166b9912d26 100644 --- a/components/eam/src/physics/crm/pam/pam_statistics.h +++ b/components/eam/src/physics/crm/pam/pam_statistics.h @@ -458,29 +458,34 @@ inline void pam_statistics_compute_means( pam::PamCoupler &coupler ) { if (clear_rh_cnt(k,iens)>0) { clear_rh(k,iens) = clear_rh(k,iens) / clear_rh_cnt(k,iens); } - phys_tend_sgs_temp (k,iens) = phys_tend_sgs_temp (k,iens) / phys_tend_sgs_cnt (iens); - phys_tend_sgs_qv (k,iens) = phys_tend_sgs_qv (k,iens) / phys_tend_sgs_cnt (iens); - phys_tend_sgs_qc (k,iens) = phys_tend_sgs_qc (k,iens) / phys_tend_sgs_cnt (iens); - phys_tend_sgs_qi (k,iens) = phys_tend_sgs_qi (k,iens) / phys_tend_sgs_cnt (iens); - phys_tend_sgs_qr (k,iens) = phys_tend_sgs_qr (k,iens) / phys_tend_sgs_cnt (iens); - - phys_tend_micro_temp(k,iens) = phys_tend_micro_temp(k,iens) / phys_tend_micro_cnt(iens); - phys_tend_micro_qv (k,iens) = phys_tend_micro_qv (k,iens) / phys_tend_micro_cnt(iens); - phys_tend_micro_qc (k,iens) = phys_tend_micro_qc (k,iens) / phys_tend_micro_cnt(iens); - phys_tend_micro_qi (k,iens) = phys_tend_micro_qi (k,iens) / phys_tend_micro_cnt(iens); - phys_tend_micro_qr (k,iens) = phys_tend_micro_qr (k,iens) / phys_tend_micro_cnt(iens); - - phys_tend_dycor_temp (k,iens) = phys_tend_dycor_temp (k,iens) / phys_tend_dycor_cnt(iens); - phys_tend_dycor_qv (k,iens) = phys_tend_dycor_qv (k,iens) / phys_tend_dycor_cnt(iens); - phys_tend_dycor_qc (k,iens) = phys_tend_dycor_qc (k,iens) / phys_tend_dycor_cnt(iens); - phys_tend_dycor_qi (k,iens) = phys_tend_dycor_qi (k,iens) / phys_tend_dycor_cnt(iens); - phys_tend_dycor_qr (k,iens) = phys_tend_dycor_qr (k,iens) / phys_tend_dycor_cnt(iens); - - phys_tend_sponge_temp(k,iens) = phys_tend_sponge_temp(k,iens) / phys_tend_sponge_cnt(iens); - phys_tend_sponge_qv (k,iens) = phys_tend_sponge_qv (k,iens) / phys_tend_sponge_cnt(iens); - phys_tend_sponge_qc (k,iens) = phys_tend_sponge_qc (k,iens) / phys_tend_sponge_cnt(iens); - phys_tend_sponge_qi (k,iens) = phys_tend_sponge_qi (k,iens) / phys_tend_sponge_cnt(iens); - phys_tend_sponge_qr (k,iens) = phys_tend_sponge_qr (k,iens) / phys_tend_sponge_cnt(iens); + if (phys_tend_sgs_cnt(iens)>0) { + phys_tend_sgs_temp (k,iens) = phys_tend_sgs_temp (k,iens) / phys_tend_sgs_cnt (iens); + phys_tend_sgs_qv (k,iens) = phys_tend_sgs_qv (k,iens) / phys_tend_sgs_cnt (iens); + phys_tend_sgs_qc (k,iens) = phys_tend_sgs_qc (k,iens) / phys_tend_sgs_cnt (iens); + phys_tend_sgs_qi (k,iens) = phys_tend_sgs_qi (k,iens) / phys_tend_sgs_cnt (iens); + phys_tend_sgs_qr (k,iens) = phys_tend_sgs_qr (k,iens) / phys_tend_sgs_cnt (iens); + } + if (phys_tend_micro_cnt(iens)>0) { + phys_tend_micro_temp(k,iens) = phys_tend_micro_temp(k,iens) / phys_tend_micro_cnt(iens); + phys_tend_micro_qv (k,iens) = phys_tend_micro_qv (k,iens) / phys_tend_micro_cnt(iens); + phys_tend_micro_qc (k,iens) = phys_tend_micro_qc (k,iens) / phys_tend_micro_cnt(iens); + phys_tend_micro_qi (k,iens) = phys_tend_micro_qi (k,iens) / phys_tend_micro_cnt(iens); + phys_tend_micro_qr (k,iens) = phys_tend_micro_qr (k,iens) / phys_tend_micro_cnt(iens); + } + if (phys_tend_dycor_cnt(iens)>0) { + phys_tend_dycor_temp (k,iens) = phys_tend_dycor_temp (k,iens) / phys_tend_dycor_cnt(iens); + phys_tend_dycor_qv (k,iens) = phys_tend_dycor_qv (k,iens) / phys_tend_dycor_cnt(iens); + phys_tend_dycor_qc (k,iens) = phys_tend_dycor_qc (k,iens) / phys_tend_dycor_cnt(iens); + phys_tend_dycor_qi (k,iens) = phys_tend_dycor_qi (k,iens) / phys_tend_dycor_cnt(iens); + phys_tend_dycor_qr (k,iens) = phys_tend_dycor_qr (k,iens) / phys_tend_dycor_cnt(iens); + } + if (phys_tend_sponge_cnt(iens)>0) { + phys_tend_sponge_temp(k,iens) = phys_tend_sponge_temp(k,iens) / phys_tend_sponge_cnt(iens); + phys_tend_sponge_qv (k,iens) = phys_tend_sponge_qv (k,iens) / phys_tend_sponge_cnt(iens); + phys_tend_sponge_qc (k,iens) = phys_tend_sponge_qc (k,iens) / phys_tend_sponge_cnt(iens); + phys_tend_sponge_qi (k,iens) = phys_tend_sponge_qi (k,iens) / phys_tend_sponge_cnt(iens); + phys_tend_sponge_qr (k,iens) = phys_tend_sponge_qr (k,iens) / phys_tend_sponge_cnt(iens); + } }); //------------------------------------------------------------------------------------------------ } diff --git a/components/eam/src/physics/crm/physpkg.F90 b/components/eam/src/physics/crm/physpkg.F90 index 33fab14066d0..6ad53894f4e3 100644 --- a/components/eam/src/physics/crm/physpkg.F90 +++ b/components/eam/src/physics/crm/physpkg.F90 @@ -638,7 +638,7 @@ subroutine phys_init( phys_state, phys_tend, pbuf2d, cam_out ) if (co2_transport()) call co2_init() call co2_diags_init(phys_state) - call gw_init() + call gw_init(pbuf2d) call rayleigh_friction_init() diff --git a/components/eam/src/physics/crm/rrtmgp/radiation.F90 b/components/eam/src/physics/crm/rrtmgp/radiation.F90 index b7b253d1b6c4..b80ea0c2fe10 100644 --- a/components/eam/src/physics/crm/rrtmgp/radiation.F90 +++ b/components/eam/src/physics/crm/rrtmgp/radiation.F90 @@ -768,6 +768,18 @@ subroutine radiation_init(state) call addfld('FLNTC'//diag(icall), horiz_only, 'A', 'W/m2', & 'Clearsky net longwave flux at top of model', & sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLUTOA'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Upwelling longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLNTOA'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Net longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLUTOAC'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Clearsky upwelling longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLNTOAC'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Clearsky net longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) call addfld('LWCF'//diag(icall), horiz_only, 'A', 'W/m2', & 'Longwave cloud forcing', & sampling_seq='rad_lwsw', flag_xyfill=.true.) @@ -2494,6 +2506,7 @@ subroutine output_fluxes_lw(icall, state, flux_all, flux_clr, qrl, qrlc) ! Working arrays real(r8), dimension(pcols,pver+1) :: flux_up, flux_dn, flux_net integer :: ncol + integer :: ktop_rad = 1 ncol = state%ncol @@ -2531,6 +2544,12 @@ subroutine output_fluxes_lw(icall, state, flux_all, flux_clr, qrl, qrlc) call outfld('FLUTC'//diag(icall), flux_clr%flux_up(1:ncol,ktop), ncol, state%lchnk) call outfld('FLDSC'//diag(icall), flux_clr%flux_dn(1:ncol,kbot+1), ncol, state%lchnk) + ! TOA fluxes (above model top, use index to rad top) + call outfld('FLUTOA'//diag(icall), flux_all%flux_up(1:ncol,ktop_rad), ncol, state%lchnk) + call outfld('FLNTOA'//diag(icall), flux_all%flux_net(1:ncol,ktop_rad), ncol, state%lchnk) + call outfld('FLUTOAC'//diag(icall), flux_clr%flux_up(1:ncol,ktop_rad), ncol, state%lchnk) + call outfld('FLNTOAC'//diag(icall), flux_clr%flux_net(1:ncol,ktop_rad), ncol, state%lchnk) + ! Calculate and output the cloud radiative effect (LWCF in history) cloud_radiative_effect(1:ncol) = flux_all%flux_net(1:ncol,ktop) - flux_clr%flux_net(1:ncol,ktop) call outfld('LWCF'//diag(icall), cloud_radiative_effect, ncol, state%lchnk) diff --git a/components/eam/src/physics/p3/eam/micro_p3.F90 b/components/eam/src/physics/p3/eam/micro_p3.F90 index e668cbaa494c..e82c4fa0a96e 100644 --- a/components/eam/src/physics/p3/eam/micro_p3.F90 +++ b/components/eam/src/physics/p3/eam/micro_p3.F90 @@ -4350,7 +4350,6 @@ subroutine ice_sedimentation(kts,kte,ktop,kbot,kdir, & dt_left, prt_accum, inv_dz, inv_rho, rho, num_arrays, vs, fluxes, qnr, dt_sub) do k = k_qxbot,k_qxtop,kdir - precip_ice_flux(k+1) = precip_ice_flux(k+1) + flux_qit(k)*dt_sub ! shanyp sflx(k+1) = sflx(k+1) + flux_qit(k)*dt_sub enddo @@ -4363,7 +4362,6 @@ subroutine ice_sedimentation(kts,kte,ktop,kbot,kdir, & bm_incld(:) = bm(:)/cld_frac_i(:) enddo substep_sedi_i - precip_ice_flux(:)=precip_ice_flux(:)*inv_dt sflx(:)=sflx(:)*inv_dt precip_ice_surf = precip_ice_surf + prt_accum*inv_rho_h2o*inv_dt diff --git a/components/eam/src/physics/rrtmgp/external b/components/eam/src/physics/rrtmgp/external index 8ff525eeed1d..b24ca1f616e4 160000 --- a/components/eam/src/physics/rrtmgp/external +++ b/components/eam/src/physics/rrtmgp/external @@ -1 +1 @@ -Subproject commit 8ff525eeed1d87a2ca6f251c4d16b46222c5554d +Subproject commit b24ca1f616e45659b334dbd7297017cb7927367e diff --git a/components/eam/src/physics/rrtmgp/radiation.F90 b/components/eam/src/physics/rrtmgp/radiation.F90 index 0c715000951d..5c87c3376d9d 100644 --- a/components/eam/src/physics/rrtmgp/radiation.F90 +++ b/components/eam/src/physics/rrtmgp/radiation.F90 @@ -767,6 +767,18 @@ subroutine radiation_init(state,pbuf) call addfld('FLNTC'//diag(icall), horiz_only, 'A', 'W/m2', & 'Clearsky net longwave flux at top of model', & sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLUTOA'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Upwelling longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLNTOA'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Net longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLUTOAC'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Clearsky upwelling longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLNTOAC'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Clearsky net longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) call addfld('LWCF'//diag(icall), horiz_only, 'A', 'W/m2', & 'Longwave cloud forcing', & sampling_seq='rad_lwsw', flag_xyfill=.true.) @@ -2375,6 +2387,7 @@ subroutine output_fluxes_lw(icall, state, flux_all, flux_clr, qrl, qrlc) ! Working arrays real(r8), dimension(pcols,pver+1) :: flux_up, flux_dn, flux_net integer :: ncol + integer :: ktop_rad = 1 ncol = state%ncol @@ -2412,6 +2425,12 @@ subroutine output_fluxes_lw(icall, state, flux_all, flux_clr, qrl, qrlc) call outfld('FLUTC'//diag(icall), flux_clr%flux_up(1:ncol,ktop), ncol, state%lchnk) call outfld('FLDSC'//diag(icall), flux_clr%flux_dn(1:ncol,kbot+1), ncol, state%lchnk) + ! TOA fluxes (above model top, use index to rad top) + call outfld('FLUTOA'//diag(icall), flux_all%flux_up(1:ncol,ktop_rad), ncol, state%lchnk) + call outfld('FLNTOA'//diag(icall), flux_all%flux_net(1:ncol,ktop_rad), ncol, state%lchnk) + call outfld('FLUTOAC'//diag(icall), flux_clr%flux_up(1:ncol,ktop_rad), ncol, state%lchnk) + call outfld('FLNTOAC'//diag(icall), flux_clr%flux_net(1:ncol,ktop_rad), ncol, state%lchnk) + ! Calculate and output the cloud radiative effect (LWCF in history) cloud_radiative_effect(1:ncol) = flux_all%flux_net(1:ncol,ktop) - flux_clr%flux_net(1:ncol,ktop) call outfld('LWCF'//diag(icall), cloud_radiative_effect, ncol, state%lchnk) diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index 14343d48f72e..cb49f41ddf25 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -461,10 +461,6 @@ if (NOT SCREAM_LIB_ONLY) # for LONG use 100. It is *completely* up to the test to decide what short, medium, and long mean. if (EKAT_ENABLE_COVERAGE OR EKAT_ENABLE_CUDA_MEMCHECK OR EKAT_ENABLE_VALGRIND OR EKAT_ENABLE_COMPUTE_SANITIZER) set (SCREAM_TEST_SIZE_DEFAULT SHORT) - # also set thread_ing=$max_thread - 1, so we test at most 2 threading configurations - if (SCREAM_TEST_MAX_THREADS GREATER 1) - math (EXPR SCREAM_TEST_THREAD_INC ${SCREAM_TEST_MAX_THREADS}-1) - endif() else() set (SCREAM_TEST_SIZE_DEFAULT MEDIUM) endif() @@ -596,9 +592,6 @@ if (NOT DEFINED ENV{SCREAM_FAKE_ONLY}) ${CMAKE_CURRENT_BINARY_DIR}/src/scream_config.h F90_FILE ${CMAKE_CURRENT_BINARY_DIR}/src/scream_config.f) - # Build any tools in the scripts/ dir. - add_subdirectory(scripts) - # Generate scream_config.h and scream_config.f include (EkatUtils) EkatConfigFile(${CMAKE_CURRENT_SOURCE_DIR}/src/scream_config.h.in diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 7e92482e6767..f7778cea5fd8 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -37,6 +37,8 @@ be lost if SCREAM_HACK_XML is not enabled. + + @@ -197,23 +199,34 @@ be lost if SCREAM_HACK_XML is not enabled. 740.0e3 ${DIN_LOC_ROOT}/atm/scream/tables/p3_lookup_table_1.dat-v4.1.1, - ${DIN_LOC_ROOT}/atm/scream/tables/mu_r_table_vals.dat8, - ${DIN_LOC_ROOT}/atm/scream/tables/revap_table_vals.dat8, - ${DIN_LOC_ROOT}/atm/scream/tables/vn_table_vals.dat8, - ${DIN_LOC_ROOT}/atm/scream/tables/vm_table_vals.dat8 + ${DIN_LOC_ROOT}/atm/scream/tables/mu_r_table_vals_v2.dat8, + ${DIN_LOC_ROOT}/atm/scream/tables/revap_table_vals_v2.dat8, + ${DIN_LOC_ROOT}/atm/scream/tables/vn_table_vals_v2.dat8, + ${DIN_LOC_ROOT}/atm/scream/tables/vm_table_vals_v2.dat8 - 1350.0 - 1.0 - 1.0 - 67.0 - 0.5 - 1.0 - 50.0 - 900.0 - 0.65 - 0.304 - 1.0 - 0.00028 + 1350.0 + 2.47 + 1.79 + 25.0e-6 + 67.0 + 1.15 + 1.15 + 5.78 + 0.00028 + 1.0 + 1.0 + 0.5 + 1.0 + 50.0 + 900.0 + 0.65 + 0.304 + 1.0 + true + + false + false + false @@ -221,7 +234,7 @@ be lost if SCREAM_HACK_XML is not enabled. false false 0.001 - 0.04 + 0.08 2.65 0.02 1.0 @@ -243,12 +256,82 @@ be lost if SCREAM_HACK_XML is not enabled. - + + ${DIN_LOC_ROOT}/atm/scream/mam4xx/drydep/ne30pg2/atmsrf_ne30pg2_c20241017.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/drydep/ne4pg2/atmsrf_ne4pg2_c20241017.nc + + + + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne120pg2_20231201.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne256pg2_20231201.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne512pg2_20231201.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne1024pg2_20231201.nc + + + + + + true + true + true + true + + 172800.0 + 3.0E-008 + 4 + 193.0 + 20100101 + ${DIN_LOC_ROOT}/atm/scream/mam4xx/linoz/ne30pg2/linoz1850-2015_2010JPL_CMIP6_10deg_58km_ne30pg2_c20240724.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/linoz/ne4pg2/linoz1850-2015_2010JPL_CMIP6_10deg_58km_ne4pg2_c20240724.nc + + 20150101 + ${DIN_LOC_ROOT}/atm/scream/mam4xx/invariants/ne30pg2/oxid_1.9x2.5_L26_1850-2015_ne30pg2_c20241009.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/invariants/ne4pg2/oxid_1.9x2.5_L26_1850-2015_ne4pg2_c20241009.nc + 20100101 + ${DIN_LOC_ROOT}/atm/scream/mam4xx/linoz/Linoz_Chlorine_Loading_CMIP6_0003-2017_c20171114.nc + + ${DIN_LOC_ROOT}/atm/scream/mam4xx/photolysis/RSF_GT200nm_v3.0_c080811.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/photolysis/temp_prs_GT200nm_JPL10_c130206.nc + + 20100101 + + ${DIN_LOC_ROOT}/atm/scream/mam4xx/drydep/season_wes.nc + + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_so2_elev_1x1_2010_clim_ne30pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_so4_a1_elev_1x1_2010_clim_ne30pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_so4_a2_elev_1x1_2010_clim_ne30pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_pom_a4_elev_1x1_2010_clim_ne30pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_bc_a4_elev_1x1_2010_clim_ne30pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_num_a1_elev_1x1_2010_clim_ne30pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_num_a2_elev_1x1_2010_clim_ne30pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_num_a4_elev_1x1_2010_clim_ne30pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_soag_elev_1x1_2010_clim_ne30pg2_c20241008.nc + + + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so2_elev_1x1_2010_clim_ne4pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so4_a1_elev_1x1_2010_clim_ne4pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so4_a2_elev_1x1_2010_clim_ne4pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_pom_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_bc_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a1_elev_1x1_2010_clim_ne4pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a2_elev_1x1_2010_clim_ne4pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_soag_elev_1x1_2010_clim_ne4pg2_c20241008.nc + + + + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne120pg2_20231201.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne256pg2_20231201.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne512pg2_20231201.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne1024pg2_20231201.nc + + + ${DIN_LOC_ROOT}/atm/scream/mam4xx/physprops/mam4_mode1_rrtmg_aeronetdust_c20240206.nc @@ -271,16 +354,7 @@ be lost if SCREAM_HACK_XML is not enabled. - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/DMSflux.2010.ne4pg2_conserv.POPmonthlyClimFromACES4BGC_c20240814.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so2_surf_ne4pg2_2010_clim_c20240815.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_bc_a4_surf_ne4pg2_2010_clim_c20240815.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a1_surf_ne4pg2_2010_clim_c20240815.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a2_surf_ne4pg2_2010_clim_c20240815.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a4_surf_ne4pg2_2010_clim_c20240815.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_pom_a4_surf_ne4pg2_2010_clim_c20240815.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a1_surf_ne4pg2_2010_clim_c20240815.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a2_surf_ne4pg2_2010_clim_c20240815.nc - + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/DMSflux.2010.ne30pg2_conserv.POPmonthlyClimFromACES4BGC_c20240816.nc ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so2_surf_ne30pg2_2010_clim_c20240816.nc ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_bc_a4_surf_ne30pg2_2010_clim_c20240816.nc @@ -291,6 +365,23 @@ be lost if SCREAM_HACK_XML is not enabled. ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so4_a1_surf_ne30pg2_2010_clim_c20240816.nc ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so4_a2_surf_ne30pg2_2010_clim_c20240816.nc + + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/DMSflux.2010.ne4pg2_conserv.POPmonthlyClimFromACES4BGC_c20240814.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so2_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_bc_a4_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a1_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a2_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a4_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_pom_a4_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a1_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a2_surf_ne4pg2_2010_clim_c20240815.nc + + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/dst_ne30pg2_c20241028.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/dst_ne4pg2_c20241028.nc + + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/monthly_macromolecules_0.1deg_bilinear_year01_merge_ne30pg2_c20241030.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/monthly_macromolecules_0.1deg_bilinear_year01_merge_ne4pg2_c20241030.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne120pg2_20231201.nc @@ -340,6 +431,9 @@ be lost if SCREAM_HACK_XML is not enabled. false + + + 1,2 @@ -357,6 +451,7 @@ be lost if SCREAM_HACK_XML is not enabled. ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne256pg2_20231201.nc ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne512pg2_20231201.nc ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne1024pg2_20231201.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30np4_to_CAx32v1pg2_intbilin_se2fv_20230420.nc @@ -397,18 +492,24 @@ be lost if SCREAM_HACK_XML is not enabled. -9999 2010 - -9999 - 0 - -9999 - 0 - -9999 - 0 + -9999.0 + 0.0 + -9999.0 + 0.0 + -9999.0 + 0.0 + + + -9999.0 + 551.58 + 1 2 4 3 3 3 + 3 3 4 true @@ -442,6 +543,7 @@ be lost if SCREAM_HACK_XML is not enabled. 6 2 1 + 1 1 5 @@ -461,6 +563,7 @@ be lost if SCREAM_HACK_XML is not enabled. mac_aero_mic,rrtmgp + iop_forcing,mac_aero_mic,rrtmgp @@ -479,9 +582,12 @@ be lost if SCREAM_HACK_XML is not enabled. UNSET - ${DIN_LOC_ROOT}/atm/scream/init/screami_ne4np4L72_20220823.nc + ${DIN_LOC_ROOT}/atm/scream/init/screami_ne4np4L72_20220823.nc + ${DIN_LOC_ROOT}/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc + ${DIN_LOC_ROOT}/atm/scream/init/screami_ne4np4L128_20241022.nc ${DIN_LOC_ROOT}/atm/scream/init/screami_ne30np4L72_20220823.nc ${DIN_LOC_ROOT}/atm/scream/init/screami_ne30np4L128_20221004.nc + ${DIN_LOC_ROOT}/atm/scream/init/screami_mam_ne30np4L72_c20240623.nc ${DIN_LOC_ROOT}/atm/scream/init/screami_ne120np4L72_20220823.nc ${DIN_LOC_ROOT}/atm/scream/init/screami_ne120np4L128_20230215.nc ${DIN_LOC_ROOT}/atm/scream/init/screami_ne256np4L128_ifs-20200120_20220914.nc @@ -490,6 +596,7 @@ be lost if SCREAM_HACK_XML is not enabled. ${DIN_LOC_ROOT}/atm/scream/init/screami_ne1024np4L128_ifs-20200120-topoadjx6t_20221011.nc ${DIN_LOC_ROOT}/atm/scream/init/screami_aquaplanet_ne4np4L72_20220823.nc ${DIN_LOC_ROOT}/atm/scream/init/screami_conusx4v1np4L72-topo12x_013023.nc + ${DIN_LOC_ROOT}/atm/scream/init/screami_CAx32v1np4L128_hiccup-era5-20150101_041823.ncpdq.nc UNSET @@ -506,8 +613,9 @@ be lost if SCREAM_HACK_XML is not enabled. UNSET 0.0 + + ${DIN_LOC_ROOT}/atm/cam/topo/USGS-gtopo30_CA_ne32_x32_v1_pg2_16xdel2.nc - ${CASE}.scream 0.0 0.0 @@ -535,9 +643,10 @@ be lost if SCREAM_HACK_XML is not enabled. 0.0 0.0,0.0 - 2.6e-08 - 0.41417721820867320E-007 - 0.15100083211582764E+004 + 1.37146e-07 ,3.45899e-08 ,1.00000e-06 ,9.99601e-08 + 1.37452e-07 ,3.46684e-08 ,1.00900e-06 ,9.99601e-08 + 5.08262e-12 ,1.54035e-13 ,3.09018e-13 ,9.14710e-22 + 0.0 0.0 0.0 0.0 @@ -576,8 +685,8 @@ be lost if SCREAM_HACK_XML is not enabled. 0.0 0.0 - 0.0 - 0.0 + 0.0 + 0.0 0.0 @@ -593,7 +702,6 @@ be lost if SCREAM_HACK_XML is not enabled. - ./${CASE}.scream default ${REST_N} @@ -606,7 +714,7 @@ be lost if SCREAM_HACK_XML is not enabled. + --> @@ -617,6 +725,7 @@ be lost if SCREAM_HACK_XML is not enabled. 0 + false @@ -677,6 +786,7 @@ be lost if SCREAM_HACK_XML is not enabled. 0 0 0 + 0 0 2 3.0 @@ -692,6 +802,7 @@ be lost if SCREAM_HACK_XML is not enabled. 4.0e4 2.0e4 1.0e4 + 1.0e4 1.0e4 100000.0 1 @@ -699,6 +810,7 @@ be lost if SCREAM_HACK_XML is not enabled. 0 0 0 + 0 0 0 sphere @@ -711,6 +823,7 @@ be lost if SCREAM_HACK_XML is not enabled. 256 512 1024 + 0 1024 0 0 @@ -732,6 +845,7 @@ be lost if SCREAM_HACK_XML is not enabled. 33.33333333333 16.6666666666666 8.3333333333333 + 8.3333333333333 8.3333333333333 75 9999 @@ -750,10 +864,12 @@ be lost if SCREAM_HACK_XML is not enabled. 6 6 12 + 0 2 none ${DIN_LOC_ROOT}/atm/cam/inic/homme/conusx4v1.g + ${DIN_LOC_ROOT}/atm/cam/inic/homme/CA_ne32_x32_v1.g diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/L128/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/L128/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/L128/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/L128/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/L72/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/L72/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/L72/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/L72/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/bfbhash/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/bfbhash/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/bfbhash/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/bfbhash/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/arm97/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/dpxx/arm97/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/arm97/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/dpxx/arm97/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/dpxx/comble/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/dpxx/comble/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/dycomsrf01/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/dpxx/dycomsrf01/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/dycomsrf01/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/dpxx/dycomsrf01/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/internal_diagnostics_level/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/internal_diagnostics_level/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/internal_diagnostics_level/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/internal_diagnostics_level/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/aci/shell_commands similarity index 77% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/aci/shell_commands index 658c94a3cf67..4928859e4e86 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/aci/shell_commands @@ -3,10 +3,7 @@ # MAM4xx adds additionaltracers to the simulation # Increase number of tracers for MAM4xx simulations #------------------------------------------------------ -$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh - -#modify initial condition file to get aerosol species ICs -$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/update_eamxx_num_tracers.sh # Add spa as RRTMG needs spa $CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,spa,rrtmgp" -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/aero_microphysics/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/aero_microphysics/shell_commands new file mode 100644 index 000000000000..81b3d44b2219 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/aero_microphysics/shell_commands @@ -0,0 +1,16 @@ + +#!/bin/sh +#------------------------------------------------------ +# MAM4xx adds additionaltracers to the simulation +# Increase number of tracers for MAM4xx simulations +#------------------------------------------------------ + +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/update_eamxx_num_tracers.sh -b + +#------------------------------------------------------ +# Add microphysics process +#------------------------------------------------------ +$CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,rrtmgp,mam4_aero_microphys" -b + + + diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/all_mam4xx_procs/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/all_mam4xx_procs/shell_commands new file mode 100644 index 000000000000..366f9afdd281 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/all_mam4xx_procs/shell_commands @@ -0,0 +1,22 @@ +#------------------------------------------------------ +# MAM4xx adds additional tracers to the simulation +# Increase the number of tracers for MAM4xx simulations +#------------------------------------------------------ +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/update_eamxx_num_tracers.sh + +#modify initial condition file to get aerosol species ICs +$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b + +# Add all MAM4 processes (except ACI) +$CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mam4_constituent_fluxes,mac_aero_mic,mam4_wetscav,mam4_optics,rrtmgp,mam4_srf_online_emiss,mam4_drydep" -b + +# Add mam4_aci in mac_aero_mic +$CIMEROOT/../components/eamxx/scripts/atmchange mac_aero_mic::atm_procs_list="tms,shoc,cldFraction,mam4_aci,p3" -b + +#Set precribed ccn to false so that P3 uses input from ACI +$CIMEROOT/../components/eamxx/scripts/atmchange p3::do_prescribed_ccn=false -b + +#Set predicted ccn to true so that P3 uses input from ACI +$CIMEROOT/../components/eamxx/scripts/atmchange p3::do_predict_nc=true -b + + diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/drydep/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/drydep/shell_commands similarity index 68% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/drydep/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/drydep/shell_commands index c5acf055a882..02b4594dbc9f 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/drydep/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/drydep/shell_commands @@ -5,12 +5,11 @@ # Increase number of tracers for MAM4xx simulations #------------------------------------------------------ -$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh -b +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/update_eamxx_num_tracers.sh -b #------------------------------------------------------ -#Update IC file and add drydep process +# Add drydep process #------------------------------------------------------ -$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b $CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,rrtmgp,mam4_drydep" -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/optics/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/optics/shell_commands similarity index 66% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/optics/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/optics/shell_commands index 224c16586104..b62620e7fe05 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/optics/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/optics/shell_commands @@ -2,9 +2,8 @@ # MAM4xx adds additionaltracers to the simulation # Increase number of tracers for MAM4xx simulations #------------------------------------------------------ -$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/update_eamxx_num_tracers.sh -$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b $CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,mam4_optics,rrtmgp" -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/remap_emiss_ne4_ne30/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/remap_emiss_ne4_ne30/shell_commands new file mode 100644 index 000000000000..9d0bbd39d976 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/remap_emiss_ne4_ne30/shell_commands @@ -0,0 +1,28 @@ + +#!/bin/sh +#------------------------------------------------------ +# MAM4xx adds additionaltracers to the simulation +# Increase number of tracers for MAM4xx simulations +#------------------------------------------------------ + +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/update_eamxx_num_tracers.sh -b + +#------------------------------------------------------ +# Add aerosol microphysics process, force ne4pg2 +# emission files and provide a ne4pg2->ne30pg2 mapping +# file +#------------------------------------------------------ +alias ATMCHANGE='$CIMEROOT/../components/eamxx/scripts/atmchange' + +ATMCHANGE physics::atm_procs_list="mac_aero_mic,rrtmgp,mam4_aero_microphys" -b + +ATMCHANGE mam4_aero_microphys::mam4_so2_elevated_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so2_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_so4_a1_elevated_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so4_a1_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_so4_a2_elevated_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so4_a2_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_pom_a4_elevated_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_pom_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_bc_a4_elevated_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_bc_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_num_a1_elevated_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a1_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_num_a2_elevated_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a2_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_num_a4_elevated_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_soag_elevated_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_soag_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::aero_microphys_remap_file='${DIN_LOC_ROOT}/atm/scream/maps/map_ne4pg2_to_ne30pg2_nco_c20241108.nc' -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands similarity index 70% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands index f47a1729ed74..249d7c2371eb 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands @@ -5,12 +5,11 @@ # Increase number of tracers for MAM4xx simulations #------------------------------------------------------ -$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh -b +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/update_eamxx_num_tracers.sh -b #------------------------------------------------------ -#Update IC file and add the processes +# Add the processes #------------------------------------------------------ -$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b $CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mam4_constituent_fluxes,mac_aero_mic,rrtmgp,mam4_srf_online_emiss" -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/update_eamxx_num_tracers.sh similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/update_eamxx_num_tracers.sh diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/wetscav/shell_commands similarity index 68% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/wetscav/shell_commands index 7b5b9a876503..d8436d38450a 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/wetscav/shell_commands @@ -4,10 +4,9 @@ # Increase number of tracers for MAM4xx simulations #------------------------------------------------------ -$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh -b +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/update_eamxx_num_tracers.sh -b #------------------------------------------------------ -#Update IC file and add wetscav process +# Add wetscav process #------------------------------------------------------ -$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b $CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,mam4_wetscav,rrtmgp" -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/README b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/README similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/README rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/README diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/diags/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/diags/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/diags/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/diags/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/hremap_to_ne4/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/hremap_to_ne4/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/hremap_to_ne4/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/hremap_to_ne4/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/phys/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/phys/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys_dyn/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/phys_dyn/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys_dyn/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/phys_dyn/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/1/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/1/shell_commands similarity index 92% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/1/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/1/shell_commands index 74bd2a4d0213..216f5c736280 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/1/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/1/shell_commands @@ -1,7 +1,7 @@ # This preset uses the three output streams (phys_dyn, phys, and diags) # It does not add remap, and uses INSTANT output -SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/output +SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output # Add the three streams . $SCRIPTS_DIR/phys/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/2/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/2/shell_commands similarity index 93% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/2/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/2/shell_commands index aea7feb5bbd6..24b71ba0aa94 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/2/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/2/shell_commands @@ -1,7 +1,7 @@ # This preset uses the three output streams (phys_dyn, phys, and diags) # It does not add remap, and uses AVERAGE output -SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/output +SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output # Add the three streams . $SCRIPTS_DIR/phys/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/3/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/3/shell_commands similarity index 93% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/3/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/3/shell_commands index da1d02dc5de1..5d27192d5254 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/3/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/3/shell_commands @@ -1,7 +1,7 @@ # This preset uses the three output streams (phys, and diags) # It adds horiz remap, and uses INSTANT output -SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/output +SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output # Add the phys/diags streams (cannot add phys_dyn, b/c we use horiz remap) . $SCRIPTS_DIR/phys/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/4/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/4/shell_commands similarity index 94% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/4/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/4/shell_commands index 521ada7f5082..6746cf1d46e9 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/4/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/4/shell_commands @@ -1,7 +1,7 @@ # This preset uses the three output streams (phys, and diags) # It adds horizontal remap, and uses AVERAGE output -SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/output +SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output # Add the phys/diags streams (cannot add phys_dyn, b/c we use horiz remap) . $SCRIPTS_DIR/phys/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/5/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/5/shell_commands similarity index 94% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/5/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/5/shell_commands index bd60a0472a87..3c63c1abd8f5 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/5/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/5/shell_commands @@ -1,7 +1,7 @@ # This preset uses the three output streams (phys_dyn, phys, and diags) # It adds vertical remap, and uses AVERAGE output -SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/output +SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output # Add the phys/dyn/diags streams . $SCRIPTS_DIR/phys/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/6/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/6/shell_commands similarity index 94% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/6/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/6/shell_commands index 937d16468516..442f8b9df8f5 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/6/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/6/shell_commands @@ -1,7 +1,7 @@ # This preset uses the three output streams (phys, and diags) # It adds horizontal and remap, and uses AVERAGE output -SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/output +SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output # Add the phys/diags streams (cannot add phys_dyn, b/c we use horiz remap) . $SCRIPTS_DIR/phys/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/set_avg/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/set_avg/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/set_avg/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/set_avg/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/vremap/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/vremap/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/vremap/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/vremap/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/perf_test/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/perf_test/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/perf_test/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/perf_test/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands index 435556401a2b..2520f7b6b12b 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands @@ -3,6 +3,7 @@ cime_root=$(./xmlquery --value CIMEROOT) input_data_dir=$(./xmlquery --value DIN_LOC_ROOT) atmchange=$cime_root/../components/eamxx/scripts/atmchange +case_name=$(./xmlquery --value CASE) # Change run length ./xmlchange RUN_STARTDATE="1994-10-01" @@ -61,7 +62,7 @@ else fi # set the output yaml files -output_yaml_files=$(find ${cime_root}/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/ -maxdepth 1 -type f) +output_yaml_files=$(find ${cime_root}/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/ -maxdepth 1 -type f) for file in ${output_yaml_files[@]}; do # if the word "coarse" is in the file name, do nothing if [[ "${file}" == *"_coarse.yaml" && "${hmapfile}" == "not-supported-yet" ]]; then @@ -82,6 +83,8 @@ for file in ${output_yaml_files[@]}; do sed -i "s|horiz_remap_file:.*_to_ne30.*|horiz_remap_file: ${hmapfile}|" ./$(basename ${file}) sed -i "s|horiz_remap_file:.*_to_DecadalSites.*|horiz_remap_file: ${armmapfile}|" ./$(basename ${file}) fi + # replace all filename prefixes so that st_archive works... + sed -i "s|eamxx_output.decadal|${case_name}.scream|" ./$(basename ${file}) done # TODO: diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyAVG_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyAVG_native.yaml similarity index 81% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyAVG_native.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyAVG_native.yaml index 4e1239b5c33c..181f841eeb81 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyAVG_native.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyAVG_native.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.1dailyAVG_native.h +filename_prefix: eamxx_output.decadal.1dailyAVG_native.h iotype: pnetcdf Averaging Type: Average Max Snapshots Per File: 1 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMAX_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyMAX_native.yaml similarity index 82% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMAX_native.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyMAX_native.yaml index a7d10efe0707..a8974c75b35e 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMAX_native.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyMAX_native.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.1dailyMAX_native.h +filename_prefix: eamxx_output.decadal.1dailyMAX_native.h iotype: pnetcdf Averaging Type: Max Max Snapshots Per File: 1 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMIN_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyMIN_native.yaml similarity index 80% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMIN_native.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyMIN_native.yaml index 653e194d278b..8d48a1bedf63 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMIN_native.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyMIN_native.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.1dailyMIN_native.h +filename_prefix: eamxx_output.decadal.1dailyMIN_native.h iotype: pnetcdf Averaging Type: Min Max Snapshots Per File: 1 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_arm.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1hourlyINST_arm.yaml similarity index 92% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_arm.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1hourlyINST_arm.yaml index 5bb07048aed6..52fc391ca4d9 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_arm.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1hourlyINST_arm.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.1hourlyINST_arm.h +filename_prefix: eamxx_output.decadal.1hourlyINST_arm.h iotype: pnetcdf Averaging Type: Instant Max Snapshots Per File: 24 # one file per day diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1hourlyINST_native.yaml similarity index 85% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_native.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1hourlyINST_native.yaml index 7a221e89f1c3..0aba4827ead7 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_native.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1hourlyINST_native.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.1hourlyINST_native.h +filename_prefix: eamxx_output.decadal.1hourlyINST_native.h iotype: pnetcdf Averaging Type: Instant Max Snapshots Per File: 24 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyAVG_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.3hourlyAVG_coarse.yaml similarity index 96% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyAVG_coarse.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.3hourlyAVG_coarse.yaml index 665294c62273..d429b11ebd1f 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyAVG_coarse.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.3hourlyAVG_coarse.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.3hourlyAVG_coarse.h +filename_prefix: eamxx_output.decadal.3hourlyAVG_coarse.h iotype: pnetcdf Averaging Type: Average Max Snapshots Per File: 8 # one file per day diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyINST_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.3hourlyINST_coarse.yaml similarity index 96% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyINST_coarse.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.3hourlyINST_coarse.yaml index 42c649545088..a2faa1b971c1 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyINST_coarse.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.3hourlyINST_coarse.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.3hourlyINST_coarse.h +filename_prefix: eamxx_output.decadal.3hourlyINST_coarse.h iotype: pnetcdf Averaging Type: Instant Max Snapshots Per File: 8 # one file per day diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyAVG_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyAVG_coarse.yaml similarity index 94% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyAVG_coarse.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyAVG_coarse.yaml index 5e4aaed07384..437142ba5599 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyAVG_coarse.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyAVG_coarse.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.6hourlyAVG_coarse.h +filename_prefix: eamxx_output.decadal.6hourlyAVG_coarse.h iotype: pnetcdf Averaging Type: Average Max Snapshots Per File: 4 # one file per day diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyINST_coarse.yaml similarity index 91% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_coarse.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyINST_coarse.yaml index e9e0f34d5e0f..bb83718a8eb0 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_coarse.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyINST_coarse.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.6hourlyINST_coarse.h +filename_prefix: eamxx_output.decadal.6hourlyINST_coarse.h iotype: pnetcdf Averaging Type: Instant Max Snapshots Per File: 4 # one file per day diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyINST_native.yaml similarity index 90% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_native.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyINST_native.yaml index bb7fd275abf4..c69dc4b2212b 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_native.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyINST_native.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.6hourlyINST_native.h +filename_prefix: eamxx_output.decadal.6hourlyINST_native.h iotype: pnetcdf Averaging Type: Instant Max Snapshots Per File: 4 # one file per day diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.dailyAVG_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.dailyAVG_coarse.yaml similarity index 97% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.dailyAVG_coarse.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.dailyAVG_coarse.yaml index 7c1990a7b56e..2d1e6e7221ef 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.dailyAVG_coarse.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.dailyAVG_coarse.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.dailyAVG_coarse.h +filename_prefix: eamxx_output.decadal.dailyAVG_coarse.h iotype: pnetcdf Averaging Type: Average Max Snapshots Per File: 1 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/rad_frequency_2/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/rad_frequency_2/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/rad_frequency_2/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/rad_frequency_2/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/scream_example_testmod_atmchange/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/scream_example_testmod_atmchange/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/scream_example_testmod_atmchange/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/scream_example_testmod_atmchange/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/scream_example_testmod_cmake/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/scream_example_testmod_cmake/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/scream_example_testmod_cmake/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/scream_example_testmod_cmake/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/sl_nsubstep2/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/sl_nsubstep2/shell_commands new file mode 100644 index 000000000000..71cdd9b691ce --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/sl_nsubstep2/shell_commands @@ -0,0 +1,3 @@ +ATMCHANGE=$CIMEROOT/../components/eamxx/scripts/atmchange + +$ATMCHANGE semi_lagrange_trajectory_nsubstep=2 -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/small_kernels/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/small_kernels/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_p3/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/small_kernels_p3/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_p3/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/small_kernels_p3/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_shoc/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/small_kernels_shoc/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_shoc/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/small_kernels_shoc/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/spa_remap/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/spa_remap/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/spa_remap/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/spa_remap/shell_commands diff --git a/components/eamxx/cime_config/tests/eamxx_default_files.py b/components/eamxx/cime_config/tests/eamxx_default_files.py deleted file mode 100644 index b39d2d5c1553..000000000000 --- a/components/eamxx/cime_config/tests/eamxx_default_files.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python3 - -import os -import http -import pathlib -import unittest -import urllib.request -import xml.etree.ElementTree as ET - - -class testNamelistDefaultsScream(unittest.TestCase): - def setUp(self): - """ - Set up the environment for the test by setting the DIN_LOC_ROOT - environment variable. Parse the 'namelist_defaults_scream.xml' - file and extract the files of interest based on the DIN_LOC_ROOT - variable or the array(file) type. Assign the extracted files - to the 'my_files' attribute of the test instance. - """ - - os.environ["DIN_LOC_ROOT"] = "https://web.lcrc.anl.gov/public/e3sm/inputdata/" - - scream_defaults_path = pathlib.Path(__file__) - tree = ET.parse(f"{scream_defaults_path.parent.parent}/namelist_defaults_scream.xml") - root = tree.getroot() - - files_of_interest = [ - child.text for child in root.findall(".//") - if child.text and child.text.startswith("${DIN_LOC_ROOT}") - ] - - more_files_of_interest = [ - child.text for child in root.findall(".//") - if child.text and "type" in child.attrib.keys() and child.attrib["type"]=="array(file)" - ] - - files_of_interest.extend( - text.strip() for text_list in more_files_of_interest for text in text_list.split(",") - if text.strip().startswith("${DIN_LOC_ROOT}") - ) - - self.my_files = [ - file.replace("${DIN_LOC_ROOT}/", "") - for file in files_of_interest - ] - - self.my_lines = [] - with open( - f"{scream_defaults_path.parent.parent}/namelist_defaults_scream.xml", - "r" - ) as the_file: - for a_line in the_file: - self.my_lines.append(a_line) - - def test_ascii_lines(self): - """ - Test that all lines are ASCII - """ - - for i_line, a_line in enumerate(self.my_lines): - with self.subTest(i_line=i_line): - self.assertTrue( - a_line.isascii(), - msg=f"\nERROR! This line is not ASCII!\n{a_line}" - ) - - def test_opening_files(self): - """ - Test the opening of files from the inputdata server. - """ - - for i_file in range(len(self.my_files)): - with self.subTest(i_file=i_file): - try: - request_return = urllib.request.urlopen( - f"{os.environ['DIN_LOC_ROOT']}{self.my_files[i_file]}" - ) - self.assertIsInstance(request_return, http.client.HTTPResponse) - except urllib.error.HTTPError: - file_name = f"{os.environ['DIN_LOC_ROOT']}{self.my_files[i_file]}" - self.assertTrue( - False, - msg=f"\nERROR! This file doesn't exist!\n{file_name}" - ) - - def test_expected_fail(self): - """ - Test an expected failure by manipulating the file name. - """ - - with self.assertRaises(urllib.error.HTTPError): - some_phony_file = f"{self.my_files[5][:-5]}some_phony_file.nc" - urllib.request.urlopen( - f"{os.environ['DIN_LOC_ROOT']}{some_phony_file}" - ) - - -if __name__ == '__main__': - unittest.main() diff --git a/components/eamxx/cmake/BuildCprnc.cmake b/components/eamxx/cmake/BuildCprnc.cmake index 2f4f1f00a362..05db9b606903 100644 --- a/components/eamxx/cmake/BuildCprnc.cmake +++ b/components/eamxx/cmake/BuildCprnc.cmake @@ -8,32 +8,46 @@ include (EkatUtils) macro(BuildCprnc) - # Make sure this is built only once - if (NOT TARGET cprnc) - if (SCREAM_CIME_BUILD) - string (CONCAT MSG - "WARNING! By default, scream should not build tests in a CIME build,\n" - "and cprnc should only be built by scream in case tests are enabled.\n" - "If you explicitly requested tests to be on in a CIME build,\n" - "then you can discard this warning. Otherwise, please, contact developers.\n") - message("${MSG}") + # TODO: handle this more carefully and more gracefully in the future + # TODO: For now, it is just a hack to get going... + # find cprnc defined in machine entries + set(CCSM_CPRNC $ENV{CCSM_CPRNC}) + if(EXISTS "${CCSM_CPRNC}") + message(STATUS "Path ${CCSM_CPRNC} exists, so we will use it") + set(CPRNC_BINARY ${CCSM_CPRNC} CACHE INTERNAL "") + configure_file (${SCREAM_BASE_DIR}/cmake/CprncTest.cmake.in + ${CMAKE_BINARY_DIR}/bin/CprncTest.cmake @ONLY) + else() + if (NOT "${CCSM_CPRNC}" STREQUAL "") + message(WARNING "Path ${CCSM_CPRNC} does not exist, so we will try to build it") endif() - set(BLDROOT ${PROJECT_BINARY_DIR}/externals/cprnc) - file(WRITE ${BLDROOT}/Macros.cmake - " - set(SCC ${CMAKE_C_COMPILER}) - set(SFC ${CMAKE_Fortran_COMPILER}) - set(FFLAGS \"${CMAKE_Fortran_FLAGS}\") - set(NETCDF_PATH ${NetCDF_Fortran_PATH}) - " - ) - set(SRC_ROOT ${SCREAM_BASE_DIR}/../..) - add_subdirectory(${SRC_ROOT}/cime/CIME/non_py/cprnc ${BLDROOT}) - EkatDisableAllWarning(cprnc) + # Make sure this is built only once + if (NOT TARGET cprnc) + if (SCREAM_CIME_BUILD) + string (CONCAT MSG + "WARNING! By default, scream should not build tests in a CIME build,\n" + "and cprnc should only be built by scream in case tests are enabled.\n" + "If you explicitly requested tests to be on in a CIME build,\n" + "then you can discard this warning. Otherwise, please, contact developers.\n") + message("${MSG}") + endif() + set(BLDROOT ${PROJECT_BINARY_DIR}/externals/cprnc) + file(WRITE ${BLDROOT}/Macros.cmake + " + set(SCC ${CMAKE_C_COMPILER}) + set(SFC ${CMAKE_Fortran_COMPILER}) + set(FFLAGS \"${CMAKE_Fortran_FLAGS}\") + set(NETCDF_PATH ${NetCDF_Fortran_PATH}) + " + ) + set(SRC_ROOT ${SCREAM_BASE_DIR}/../..) + add_subdirectory(${SRC_ROOT}/cime/CIME/non_py/cprnc ${BLDROOT}) + EkatDisableAllWarning(cprnc) - set(CPRNC_BINARY ${BLDROOT}/cprnc CACHE INTERNAL "") + set(CPRNC_BINARY ${BLDROOT}/cprnc CACHE INTERNAL "") - configure_file (${SCREAM_BASE_DIR}/cmake/CprncTest.cmake.in - ${CMAKE_BINARY_DIR}/bin/CprncTest.cmake @ONLY) + configure_file (${SCREAM_BASE_DIR}/cmake/CprncTest.cmake.in + ${CMAKE_BINARY_DIR}/bin/CprncTest.cmake @ONLY) + endif() endif() endmacro() diff --git a/components/eamxx/cmake/machine-files/gcp.cmake b/components/eamxx/cmake/machine-files/gcp12.cmake similarity index 100% rename from components/eamxx/cmake/machine-files/gcp.cmake rename to components/eamxx/cmake/machine-files/gcp12.cmake diff --git a/components/eamxx/cmake/machine-files/ghci-oci.cmake b/components/eamxx/cmake/machine-files/ghci-oci.cmake index 86a2fb1d5302..85eabbaa848a 100644 --- a/components/eamxx/cmake/machine-files/ghci-oci.cmake +++ b/components/eamxx/cmake/machine-files/ghci-oci.cmake @@ -1,2 +1,15 @@ include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) common_setup() + +set(CMAKE_Fortran_FLAGS "-Wno-maybe-uninitialized -Wno-unused-dummy-argument -fallow-argument-mismatch" CACHE STRING "" FORCE) +set(CMAKE_CXX_FLAGS "-fvisibility-inlines-hidden -fmessage-length=0 -Wno-use-after-free -Wno-unused-variable -Wno-maybe-uninitialized" CACHE STRING "" FORCE) + +# TODO: figure out a better way to handle this, e.g., +# TODO: --map-by ppr:1:node:pe=1 doesn't work with mpich, +# TODO: but -map-by core:1:numa:hwthread=1 may work well? +# TODO: this will need to be handled in EKAT at some point +set(EKAT_MPI_NP_FLAG "-np" CACHE STRING "-np") + +# TODO: hack in place to get eamxx to recognize CPRNC +# TODO: See note in BuildCprnc.cmake... +set(ENV{CCSM_CPRNC} "/usr/local/packages/bin/cprnc") diff --git a/components/eamxx/cmake/machine-files/ghci-snl-cpu.cmake b/components/eamxx/cmake/machine-files/ghci-snl-cpu.cmake new file mode 100644 index 000000000000..2fd6a9cbb426 --- /dev/null +++ b/components/eamxx/cmake/machine-files/ghci-snl-cpu.cmake @@ -0,0 +1,5 @@ +# Common settings for our ghci images +include(${CMAKE_CURRENT_LIST_DIR}/ghci-snl.cmake) + +# Set SCREAM_MACHINE +set(SCREAM_MACHINE ghci-snl-cpu CACHE STRING "") diff --git a/components/eamxx/cmake/machine-files/ghci-snl-cuda.cmake b/components/eamxx/cmake/machine-files/ghci-snl-cuda.cmake new file mode 100644 index 000000000000..83d1d20c6279 --- /dev/null +++ b/components/eamxx/cmake/machine-files/ghci-snl-cuda.cmake @@ -0,0 +1,17 @@ +# Common settings for our ghci images +include(${CMAKE_CURRENT_LIST_DIR}/ghci-snl.cmake) + +# Set SCREAM_MACHINE +set(SCREAM_MACHINE ghci-snl-cuda CACHE STRING "") + +# Enable CUDA in kokkos +set (EKAT_MACH_FILES_PATH ${CMAKE_CURRENT_LIST_DIR}/../../../../externals/ekat/cmake/machine-files) +include (${EKAT_MACH_FILES_PATH}/kokkos/cuda.cmake) + +set(EKAT_MPI_NP_FLAG "-n" CACHE STRING "The mpirun flag for designating the total number of ranks") + +# TODO: rebuild cuda image with cuda-aware MPI, so we can set this to ON +option(SCREAM_MPI_ON_DEVICE "Whether to use device pointers for MPI calls" OFF) + +# Currently, we have 2 GPUs/node on Blake, and we run a SINGLE build per node, so we can fit 2 ranks there +set(SCREAM_TEST_MAX_RANKS 2 CACHE STRING "Upper limit on ranks for mpi tests") diff --git a/components/eamxx/cmake/machine-files/ghci-snl-openmp.cmake b/components/eamxx/cmake/machine-files/ghci-snl-openmp.cmake deleted file mode 100644 index 3139580e9ee0..000000000000 --- a/components/eamxx/cmake/machine-files/ghci-snl-openmp.cmake +++ /dev/null @@ -1,9 +0,0 @@ -# Common settings for our ghci images -include(${CMAKE_CURRENT_LIST_DIR}/ghci-snl.cmake) - -# Set SCREAM_MACHINE -set(SCREAM_MACHINE ghci-snl-openmp CACHE STRING "") - -# Set OpenMP backend -set(EKAT_MACH_FILES_PATH ${CMAKE_CURRENT_LIST_DIR}/../../../../externals/ekat/cmake/machine-files) -include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) diff --git a/components/eamxx/cmake/machine-files/ghci-snl.cmake b/components/eamxx/cmake/machine-files/ghci-snl.cmake index f8115aec966b..9782036d92c9 100644 --- a/components/eamxx/cmake/machine-files/ghci-snl.cmake +++ b/components/eamxx/cmake/machine-files/ghci-snl.cmake @@ -13,3 +13,9 @@ option (Kokkos_ENABLE_DEPRECATED_CODE_4 "" OFF) # We need to manage resources to spread across available cores/gpus option (EKAT_TEST_LAUNCHER_MANAGE_RESOURCES "" ON) + +# Needed by EkatCreateUnitTest +set (EKAT_MPIRUN_EXE "mpirun" CACHE STRING "") +set (EKAT_MPI_NP_FLAG "-n" CACHE STRING "") + +set(EKAT_VALGRIND_SUPPRESSION_FILE "/projects/e3sm/baselines/scream/ghci-snl-cpu/eamxx-valgrind.supp" CACHE FILEPATH "Use this valgrind suppression file if valgrind is enabled.") diff --git a/components/eamxx/docker/Dockerfile b/components/eamxx/docker/Dockerfile deleted file mode 100644 index 4dcdc8371e0d..000000000000 --- a/components/eamxx/docker/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -ARG CIME_VERSION=latest -FROM cime:${CIME_VERSION} AS base - -ENV USER=root -ENV LOGNAME=root -ENV DEBIAN_FRONTEND="noninteractive" - -SHELL ["/bin/bash", "-c"] - -# Install additional packages -RUN mamba install --yes -c conda-forge \ - lapack \ - blas && \ - rm -rf /opt/conda/pkgs/* - -# Install dependencies -COPY components/eamxx/docker/requirements.txt /tmp/requirements.txt -RUN pip install -r /tmp/requirements.txt - -# install gcloud -RUN apt-get update && apt-get install -y apt-transport-https ca-certificates gnupg curl gettext - -RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list &&\ - curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - - -RUN apt-get update && apt-get install -y google-cloud-sdk -RUN gcloud config set project vcm-ml - -ENV OMPI_ALLOW_RUN_AS_ROOT=1 -ENV OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 -ENV IS_DOCKER=TRUE - -COPY / /src/E3SM \ No newline at end of file diff --git a/components/eamxx/docker/README.md b/components/eamxx/docker/README.md deleted file mode 100644 index a5e92f819977..000000000000 --- a/components/eamxx/docker/README.md +++ /dev/null @@ -1,24 +0,0 @@ -SCREAM Docker -====== - -SCREAM Docker is built on top of CIME's Docker image. Make sure CIME image is built first, please refer to the CIME's [documentation](../../../cime/docker/README.md) for more information on building the base image. - -## Building the container - -From the top level of the SCREAM repository, run the following commands: -```bash -docker build -t cime:latest --target base cime/docker/ -docker build --file components/eamxx/docker/Dockerfile -t scream:latest . -``` - -## Running the container - -```bash -docker run -it --hostname docker -e CIME_MODEL=e3sm scream:latest bash -``` - -It is highly recommended to mount your current working scream repository to the container: - -```bash -docker run -it -v $SCREAM_REPO:/src/E3SM --hostname docker -e CIME_MODEL=e3sm scream:latest bash -``` diff --git a/components/eamxx/docker/requirements.txt b/components/eamxx/docker/requirements.txt deleted file mode 100644 index 85388d5778c8..000000000000 --- a/components/eamxx/docker/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -numpy -xarray -zarr -netCDF4 -cftime \ No newline at end of file diff --git a/components/eamxx/docs/developer/ci_nightly.md b/components/eamxx/docs/developer/ci_nightly.md index 4089f523daf9..0716ce4c9f3e 100644 --- a/components/eamxx/docs/developer/ci_nightly.md +++ b/components/eamxx/docs/developer/ci_nightly.md @@ -1,4 +1,17 @@ # Continuous Integration and Nightly Testing -* Autotester quick overview -* Nightly overview, CDash +## Autotester + +EAMxx using github actions and a Sandia product called Autotester 2 +to run CI testing on a CPU and GPU machine for every github pull +request. By default, we run the e3sm_scream_v1_at suite and the +standalone eamxx tests (test-all-scream). + +## Nightly overview, CDash + +Our nightly testing is much more extensive than the CI testing. You +can see our dashboard here under the section "E3SM_SCREAM": + + +We run a variety of CIME test suites and standalone testing on a number +of platforms. We even do some performance testing on frontier. diff --git a/components/eamxx/docs/developer/cime_testing.md b/components/eamxx/docs/developer/cime_testing.md index 7c2d91a36579..667488960f64 100644 --- a/components/eamxx/docs/developer/cime_testing.md +++ b/components/eamxx/docs/developer/cime_testing.md @@ -1,7 +1,68 @@ # Full Model Testing -Quickly review CIME test infrastructure and how EAMxx uses it +Full model system testing of eamxx is done through CIME test cases +(much like the rest of E3SM). -* test types, specifiers (`_LnX`,`_D`,`_PMxN`,..), grids, compsets, test-mods -* available grids/compsets for EAMxx, and where to find them -* how to add atmchange in `shell_commands` test mods +We offer a number of test suites, including: + +* e3sm_scream_v0: Test the full set of V0 (pre-C++) tests +* e3sm_scream_v1: Test the full set of V1 (C++) tests +* e3sm_scream_v1_at: A smaller and quicker set of tests for autotesting +* e3sm_scream_hires: A small number of bigger, longer-running tests to measure performance + +Example for running a suite: + +```shell +cd $repo/cime/scripts +./create_test e3sm_scream_v1_at --wait +``` + +Example for running a single test case: + +```shell +cd $repo/cime/scripts +./create_test SMS.ne4_ne4.F2010-SCREAMv1 --wait +``` + +There are many behavioral tweaks you can make to a test case, like +changing the run length, test type, etc. Most of this is not specific +to eamxx and works for any CIME case. This generic stuff +is well-documentated here: + + +When it comes to things specific to eamxx, you have grids, compsets, and +testmods. + +Common EAMxx grids are: + +* ne4_ne4 (low resolution) +* ne4pg2_ne4pg2 (low resolution with phys grid) +* ne30_ne30 (med resolution) +* ne30pg2_ne30pg2 (med resolution with phys grid) +* ne1024pg2_ne1024pg2 (ultra high with phys grid) + +More grid info can be found here: + + +Common EAMxx compsets are: + +* F2010-SCREAM-LR: V0 low res compset with eamxx V0 atmosphere +* F2010-SCREAMv1: V1 standard compset with eamxx V1 atmosphere +* FIOP-SCREAMv1-DP: V1 with dpxx (doubly-periodic lateral boundary condition in C++) +* F2010-SCREAMv1-noAero: V1 without aerosol forcing + +Full info on supported compsets can be found by looking at this file: +`$scream_repo/components/eamxx/cime_config/config_compsets.xml` + +Common EAMxx testmods are: + +* small_kernels: Enable smaller-granularity kernels, + can improve performance on some systems +* scream-output-preset-[1-6]: Our 6 output presets. + These turn some combination of our three output streams + (phys_dyn, phys, and diags), + various remaps, etc. +* bfbhash: Turns on bit-for-bit hash output: + +More info on running EAMxx can be found here: + diff --git a/components/eamxx/docs/developer/field.md b/components/eamxx/docs/developer/field.md index 013dcf857b62..8df83440a2fd 100644 --- a/components/eamxx/docs/developer/field.md +++ b/components/eamxx/docs/developer/field.md @@ -1,46 +1,58 @@ -## Field +# Field -In EAMxx, a `Field` is a data structure holding two things: pointers to the data and pointers to metadata. -Both the data and metadata are stored in `std::shared_ptr` instances, to ensure consistency across all copies -of the field. This allows for fast shallow copy semantic for this class. +In EAMxx, a `Field` is a data structure holding two things: pointers to the +data and pointers to metadata. Both the data and metadata are stored in +`std::shared_ptr` instances, to ensure consistency across all copies of +the field. This allows for fast shallow copy semantic for this class. -The data is stored on both CPU and device memory (these may be the same, depending on the Kokkos -backend). In EAMxx, we always assume and guarantee that the device data is up to date. That implies that the data -be explicitly synced to host before using it on host, and explicitly synced to device after host manipulation, -in order to ensure correctness. In order to access the data, users must use the `get_view` method, which takes -two template arguments: the data type, and an enum specifying whether CPU or device data is needed. The data -type is used to reinterpret the generic pointer stored inside to a view of the correct scalar type and layout. -It is a possibly const-qualified type, and if the field was marked as "read-only", the method ensures that the -provided data type is const. A read-only field can be created via the `getConst` method, which returns an -identical copy of the field, but marked as read-only. The enum specifying host or device data is optional, -with device being the default. +The data is stored on both CPU and device memory (these may be the same, +depending on the Kokkos backend). In EAMxx, we always assume and guarantee +that the device data is up to date. That implies that the data must be +explicitly synced to host before using it on host, and explicitly synced +to device after host manipulation, in order to ensure correctness. +In order to access the data, users must use the `get_view`/ +`get_strided_view` methods, which takes two template arguments: +the data type, and an enum specifying whether CPU or device data is needed. +The data type is used to reinterpret the generic pointer stored inside +to a view of the correct scalar type and layout. It is a possibly +const-qualified type, and if the field was marked as "read-only", +the method ensures that the provided data type is const. A read-only field +can be created via the `getConst` method, which returns a shallow copy of +the field, but marked as read-only. The enum specifying host or device data +is optional, with device being the default. -The metadata is a collection of information on the field, such as name, layout, units, allocation size, and more. -Part of the metadata is immutable after creation (e.g., name, units, or layout), while some metadata can be -partially or completely modified. The metadata is contained in the `FieldHeader` data structure, which contains -four parts: +The metadata is a collection of information on the field, such as name, layout, units, +allocation size, and more. Part of the metadata is immutable after creation (e.g., +name, units, or layout), while some metadata can be partially or completely modified. +The metadata is contained in the `FieldHeader` data structure, which contains four +parts: -* `FieldIdentifier`: stores the field's name, layout, units, data type, and name of the grid where it's defined. - These information are condensed in a single string, that can be used to uniquely identify a field, - allowing to distinguish between different version of the same field. The layout is stored in the `FieldLayout` - data structure, which includes: - * the field tags: stored as a `std::vector`, they give context to the field's extents. - * the field dims: stored both as a `std::vector`, as well as a 1d `Kokkos::View`. -* `FieldTracking`: stores information on the usage of the field, as well as its possible connections to other - fields. In particular, the tracked items are: - * the field time stamp: the time stamp when the field was last updated. - * the field accumulation start time: used for fields that are accumulated over several time steps - (or time step subcycles). For instance, it allows to reconstruct fluxes from raw accumulations. - * the providers/customers: lists of atmosphere processes (see below) that respectively require/compute - the field in their calculations. - * the field groups: a list of field groups that this field belongs too. Field groups are used to access - a group of fields without explicit prior knowledge about the number and/or names of the fields. -* `FieldAllocProp`: stores information about the allocation. While the field is not yet allocated, users can - request special allocations for the field, for instance to accommodate packing (for SIMD), which may - require padding. Upon allocation, this information is then used by the Field structure to extract the - actual data, wrapped in a properly shaped `Kokkos::View`. The alloc props are also responsible of tracking - additional information in case the field is a "slice" of a higher-dimensional one, a fact that can affect - how the data is accessed. -* Extra data: stored as a `std::map`, allows to catch any metadata that does not fit - in the above structures. This is a last resort structure, intended to accommodate the most peculiar - corner cases, and should be used sparingly. +* `FieldIdentifier`: stores the field's name, layout, units, data type, + and name of the grid where it's defined. These information are condensed + in a single string, that can be used to uniquely identify a field, allowing + to distinguish between different version of the same field. + The layout is stored in the `FieldLayout` data structure, which includes: + * the field tags: stored as a `std::vector`, they give context to the + field's extents. + * the field dims: stored both as a `std::vector`, as well as a 1d `Kokkos::View`. +* `FieldTracking`: stores information on the usage of the field, as well as its + possible connections to other fields. In particular, the tracked items are: + * the field time stamp: the time stamp when the field was last updated. + * the field accumulation start time: used for fields that are accumulated over + several time steps (or time step subcycles). For instance, it allows to + reconstruct fluxes from raw accumulations. + * the providers/customers: lists of atmosphere processes (see below) that + respectively require/compute the field in their calculations. + * the field groups: a list of field groups that this field belongs too. Field groups + are used to access a group of fields without explicit prior knowledge about the + number and/or names of the fields. +* `FieldAllocProp`: stores information about the allocation. While the field is not + yet allocated, users can request special allocations for the field, for instance + to accommodate packing (for SIMD), which may require padding. Upon allocation, + this information is then used by the Field structure to extract the actual data, + wrapped in a properly shaped `Kokkos::View`. The alloc props are also + responsible of tracking additional information in case the field is a "slice" of + a higher-dimensional one, a fact that can affect how the data is accessed. +* Extra data: stored as a `std::map`, allows to catch any + metadata that does not fit in the above structures. This is a last resort structure, + intended to accommodate the most peculiar corner cases, and should be used sparingly. diff --git a/components/eamxx/docs/developer/grid.md b/components/eamxx/docs/developer/grid.md index 8a61b97e0795..b4e1a1c8c033 100644 --- a/components/eamxx/docs/developer/grid.md +++ b/components/eamxx/docs/developer/grid.md @@ -1,22 +1,29 @@ -## Grids and Remappers +# Grids and Remappers -In EAMxx, the `AbstractGrid` is an interface used to access information regarding the horizontal and vertical -discretization. The most important information that the grid stores is: +In EAMxx, the `AbstractGrid` is an interface used to access information regarding +the horizontal and vertical discretization. The most important information that +the grid stores is: -* the number of local/global DOFs: these are the degrees of freedom of the horizontal grid only. Here, - local/global refers to the MPI partitioning. -* the DOFs global IDs (GIDs): a list of GIDs of the DOFs on the current MPI rank, stored as a Field -* the local IDs (LIDs) to index list: this list maps the LID of a DOF (that is, the position of the DOF - in the GID list) to a "native" indexing system for that DOF. For instance, a `PointGrid` (a class derived from - `AbstractGrid`) is a simple collection of points, so the "native" indexing system coincides with the LIDs. - However, for a `SEGrid` (a derived class, for spectral element grids), the "native" indexing is a triplet - `(ielem,igp,jgp)`, specifying the element index, and the two indices of the Gauss point within the element. -* geometry data: stored as a `std::map`, this represent any data that is intrinsically - linked to the grid (either along the horizontal or vertical direction), such as lat/lon coordinates, - vertical coordinates, area associated with the DOF. +* the number of local/global DOFs: these are the degrees of freedom of the + horizontal grid only. Here, local/global refers to the MPI partitioning. +* the DOFs global IDs (GIDs): a list of GIDs of the DOFs on the current MPI rank, + stored as a Field +* the local IDs (LIDs) to index list: this list maps the LID of a DOF (that is, + the position of the DOF in the GID list) to a "native" indexing system for that + DOF. For instance, a `PointGrid` (a class derived from `AbstractGrid`) is a + simple collection of points, so the "native" indexing system coincides with the + LIDs. However, for a `SEGrid` (a derived class, for spectral element grids), + the "native" indexing is a triplet `(ielem,igp,jgp)`, specifying the element + index, and the two indices of the Gauss point within the element. +* geometry data: stored as a `std::map`, this represent any + data that is intrinsically linked to the grid (either along the horizontal or + vertical direction), such as lat/lon coordinates, vertical coordinates, area + associated with the DOF. -Grids can also be used to retrieve the layout of a 2d/3d scalar/vector field, which allows certain downstream -classes to perform certain operations without assuming anything on the horizontal grid. +Grids can also be used to retrieve the layout of a 2d/3d scalar/vector field, +which allows certain downstream classes to perform certain operations without +assuming anything on the horizontal grid. -In general, grid objects are passed around the different parts of EAMxx as const objects (read-only). -The internal data can only be modified during construction, which usually is handled by a `GridsManager` object. +In general, grid objects are passed around the different parts of EAMxx as const +objects (read-only). The internal data can only be modified during construction, +which usually is handled by a `GridsManager` object. diff --git a/components/eamxx/docs/developer/index.md b/components/eamxx/docs/developer/index.md index 2d47bab65fe3..7216d6057cb4 100644 --- a/components/eamxx/docs/developer/index.md +++ b/components/eamxx/docs/developer/index.md @@ -1,3 +1,16 @@ -# SCREAM Developer Guide - +# EAMxx Developer Guide +### [Installation](../common/installation.md) +### [Style Guide](style_guide.md) +### [Kokkos and EKAT](kokkos_ekat.md) +### [Source Tree](source_tree.md) +### Important Data Structures + * [Fields](field.md) + * [Grids and Remappers](grid.md) + * [Atmosphere Processes](processes.md) + * [Managers](managers.md) +### [I/O](io.md) +### Testing + * [Standalone](standalone_testing.md) + * [Full Model](cime_testing.md) + * [CI and Nightly Testing](ci_nightly.md) diff --git a/components/eamxx/docs/developer/io.md b/components/eamxx/docs/developer/io.md index caf237010a33..0a4c7b2d8323 100644 --- a/components/eamxx/docs/developer/io.md +++ b/components/eamxx/docs/developer/io.md @@ -1,5 +1,5 @@ # Input-Output -In EAMxx, I/O is handled through the SCORPIO library, currently a submodule of E3SM. -The `scream_io` library within eamxx allows to interface the EAMxx infrastructure classes -with the SCORPIO library. +In EAMxx, I/O is handled through the SCORPIO library, currently a submodule of +E3SM. The `scream_io` library within eamxx allows to interface the EAMxx +infrastructure classes with the SCORPIO library. diff --git a/components/eamxx/docs/developer/kokkos_ekat.md b/components/eamxx/docs/developer/kokkos_ekat.md index 4a5df20ab80a..6ebf23ba1aac 100644 --- a/components/eamxx/docs/developer/kokkos_ekat.md +++ b/components/eamxx/docs/developer/kokkos_ekat.md @@ -1,12 +1,251 @@ -# Building Blocks +# Kokkos and EKAT -Here we can discuss EKAT, Kokkos, and all of the highly-technical non-scientific -stuff that makes our heads hurt. +## Kokkos -## Kokkos Views +EAMxx uses Kokkos for performance portable abstractions for parallel execution +of code and data management to various HPC platforms, including OpenMP, Cuda, +HIP, and SYCL. Here we give a brief overview of the important concepts for +understanding Kokkos in EAMxx. For a more in depth description, see the +[Kokkos wiki](https://kokkos.org/kokkos-core-wiki). -## Vectorization: Packs +### Kokkos::Device -## Fields and the Field Manager +`Kokkos::Device` is a struct which contain the type definitions for two main +Kokkos concepts: execution space (`Kokkos::Device::execution_space`), the place +on-node where parallel operations (like for-loops, reductions, etc.) are +executed, and the memory space (`Kokkos::Device::memory_space`), the memory +location on-node where data is stored. Given your machine architecture, Kokkos +defines a default "device" space, given by -### Preconditions, Postconditions, and Invariants +```cpp +Kokkos::Device +``` + +where all performance critical code should be executed (e.g., on an NVIDIA +machine, this device would be the GPU accelerators) and a default "host" space, +given by + +```c++ +Kokkos::Device +``` + +where data can be accessed by the CPU cores and is necessary for I/O +interfacing, for example. Currently, these default spaces are the ones used by +EAMxx. On CPU-only machines, host and device represent the same space. + +### Kokkos Views + +The main data struct provided by Kokkos used in EAMxx in the `Kokkos::View`. +This is a multi-dimensional data array that can live on either device or host +memory space. These Views are necessary when running on GPU architectures as +data structures like `std::vector` and `std::array` will be unavailable on +device. + +Views are constructed in EAMxx most commonly with the following template and +input arguments + +```cpp +Kokkos::View(const std::string& label, + int dim0, int dim1, ...) +``` + +where + +- `DataType`: scalar type of the view, given as `ScalarType`+`*`(x's number of + run-time dimensions). E.g., a 2D view of doubles will have `DataType = + double**`. There is also an ability to define compile-time dimensions by + using `[]`, see + [Kokkos wiki section on views] + ( temperature( + "temperature", ncols, nlevs); +``` + +### Deep Copy + +Kokkos provides `Kokkos::deep_copy(dst, src)` which copies data between views +of the same dimensions, or a scalar values into a view. Common uses + +```cpp +Kokkos::deep_copy(view0, view1); // Copy all data from view1 into view0 +Kokkos::deep_copy(view0, 5); // Set all values of view0 to 5 +``` + +As seen in the next section, we can use `deep_copy()` to copy data between host +and device. + +### Mirror Views + +We will often need to have memory allocation the resides on device (for +computation), and then need that identical data on host (say, for output). +Kokkos has a concept of mirror views, where data can be copied from host to +device and vice versa. + +Here is an example using the device view `temperature` from above + +```cpp +// Allocate view on host that exactly mirrors the size of layout of the device +view +auto host_temperature = Kokkos::create_mirror_view(temperature); + +// Copy all data from device to host +Kokkos::deep_copy(host_temperature, temperature); +``` + +Kokkos also offers an all-in-one option + +```cpp +// Note: must hand the host device instance as first argument +auto host_temperature = Kokkos::create_mirror_view_and_copy( + Kokkos::DefaultHostDevice(), temperature); +``` + +### Parallel Execution + +The most basic parallel execution pattern used by EAMxx is the +`Kokkos::parallel_for` which defines a for-loop with completely independent +iterations. The `parallel_for` takes in an optional label for debugging, an +execution policy, which defines a range and location (host or device) for the +code to be run, and a lambda describing the body of code to be executed. The +following are execution policies used in EAMxx + +- `int count`: 1D iteration range `[0, count)` +- `RangePolicy(int beg, int end)`: 1D iteration range for indices + `[beg, end)` +- `MDRangePolicy>(int[N] beg, int[N] end)`: multi- + dimensional iteration range `[beg, end)` +- `TeamPolicy(int league_size, int team_size, int vector_size)`: 1D + iteration over `league_size`, assigned to thread teams of size `team_size`, + each with `vector_size` vector lanes. Both `team_size` and `vector_size` can + be given `Kokkos::AUTO` as input for Kokkos to automatically compute. + +If no `ExecSpace` template is given, the default execution space is used. + +For lambda capture, use `KOKKOS_LAMBDA` macro which sets capture automatically +based on architecture. + +Example using `RangePolicy` to initialize a view + +```cpp +Kokkos::View temperature("temperature", ncols, + nlevs); +Kokkos::parallel_for("Init_temp", + Kokkos::RangePolicy(0, ncols*nlevs), + KOKKOS_LAMBDA (const int idx) { + int icol = idx/nlevs; + int ilev = idx%nlevs; + + temperature(icol, ilev) = 0; +}); +``` + +Same example with `TeamPolicy` + +```cpp +Kokkos::parallel_for("Init_temp", + Kokkos::TeamPolicy(ncols*nlevs, Kokkos::AUTO, Kokkos::AUTO), + KOKKOS_LAMBDA (const TeamPolicy::member_type& team) { + // league_rank() gives the index for this team + int icol = team.league_rank()/nlevs; + int ilev = team.league_rank()%nlevs; + + temperature(icol, ilev) = 0; +}); +``` + +### Hierarchical Parallelism + +Using `TeamPolicy`, we can have up to three nested levels of parallelism: team +parallelism, thread parallelism, vector parallelism. These nested policies can +be called within the lambda body using the following execution policies + +- `TeamThreadRange(team, begin, end)`: execute over threads of a team +- `TeamVectorRange(team, begin, end)`: execute over threads and vector lanes of + a team +- `ThreadVectorRange(team, begin, end)`: execute over vector lanes of a thread + +An example of using these policies + +```cpp +Kokkos::View Q("tracers", ncols, ntracers, nlevs); +Kokkos::parallel_for(Kokkos::TeamPolicy(ncols, Kokkos::AUTO), + KOKKOS_LAMBDA (TeamPolicy::member_type& team) { + int icol = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlevs), [&](int ilev) { + temperature(icol, ilev) = 0; + }); + + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nlevs), [&](int ilev) { + Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, ntracers), [&](int iq) { + Q(icol, iq, ilev) = 0; + }); + }); +}); +``` + +IMPORTANT! Nested policies cannot be used in arbitrary order. `ThreadVectorRange` +must be used inside a `TeamThreadRange`, and `TeamVectorRange` must be the only +level of nested parallelism. + +```cpp +Kokkos::parallel_for(TeamPolicy(...), ... { + // OK + Kokkos::parallel_for(TeamThreadRange, ... { + + }); + + // OK + Kokkos::parallel_for(TeamThreadRange, ... { + Kokkos::parallel_for(ThreadVectorRange, ... { + + }); + }); + + // OK + Kokkos::parallel_for(TeamVectorRange, ...{ + + }); + + // WRONG,ThreadVectorRange must be nested in TeamThreadRange + Kokkos::parallel_for(ThreadVectorRange, ...{ + + }); + + // WRONG, a TeamVectorRange must be the only nested level + Kokkos::parallel_for(TeamVectorRange, ...{ + Kokkos::parallel_for(ThreadVectorRange, ... { + + }); + }); +}); +``` + +Using these incorrectly can be very tricky to debug as the code almost certainly +will _not_ error out, but race conditions will exist among threads. + +## EKAT + +### KokkosTypes + +### ExeSpaceUtils + +### Vectorization: Packs + +### Scratch Memory: WorspaceManager + +### Algorithms diff --git a/components/eamxx/docs/developer/managers.md b/components/eamxx/docs/developer/managers.md index 676449a21845..fa98c8b1d720 100644 --- a/components/eamxx/docs/developer/managers.md +++ b/components/eamxx/docs/developer/managers.md @@ -1 +1 @@ -## FieldManager and GridsManager +# FieldManager and GridsManager diff --git a/components/eamxx/docs/developer/processes.md b/components/eamxx/docs/developer/processes.md index d6a81b3cae20..a2f2d671e2e6 100644 --- a/components/eamxx/docs/developer/processes.md +++ b/components/eamxx/docs/developer/processes.md @@ -1,18 +1,233 @@ # Atmospheric Processes -In EAMxx, the `AtmosphereProcess` (AP) is a class representing a portion of the atmosphere timestep algorithm. -In simple terms, an AP is an object that given certain input fields performs some calculations to compute -some output fields. - -TODO: describe init sequcene (e.g., the process of requesting fields), base class main - interfaces/capabilities (e.g., subcycling), class expectations (e.g., must update fields on physics grid) - -Here is a list of currently implemented atmosphere processes. -TODO: add links to papers/github-repos, and a SMALL description -* p3: Microphysics, blah blah -* SHOC: Macrophysics/Turbulence, blah -* rrtmgp: Radiation, blah -* spa: prescribed aerosols, blah blah -* surface coupling: blah -* mam: prognostic aerosols, blah blah -* nudging: This process is responsible for nudging the model simulation given a set of files with a target nudged state. +In EAMxx, `AtmosphereProcess` (AP) is an abstract class representing a portion +of the atmosphere timestep algorithm. In simple terms, an AP is an object that +given certain input fields performs some calculations to compute some output +fields. The concrete AP classes allow to create a buffer layer between +particular packages (e.g., dynamics dycore, physics parametrizations) and the +atmosphere driver (AD), allowing separation of concerns, so that the AD does +not need to know details about the package, and the package does not need to +know about the EAMxx infrastructure. + +To enhance this separation of concerns, EAMxx implements two more classes for +handling APs: + +- the concrete class `AtmosphereProcessGroup` (APG), which allows to group + together a set of AP's, which can be seen from outside as a single process; +- the `AtmosphereProcessFactory` class, which allows an APG to create its + internal processes without any knowledge of what they are. + +This infrastructure allows the AD to view the whole atmosphere as a single APG, +and to be completely agnostic to what processes are run, and in which order. +This design allows to have a code base that is cleaner, self-container, and +easy to test via a battery of targeted unit tests. + +In EAMxx, we already have a few concrete AP's, interfacing the AD to the +Hommexx non-hydrostatic dycore as well as some physics parametrizations (P3, +SHOC, RRMTPG, etc). In the next section we describe the interfaces of an AP +class, and we show an example of how to write a new concrete AP class. + +## Atmosphere process interfaces + +An AP has several interfaces, which can be grouped into three categories: + +- initialization: these interfaces are used to create the AP, as well as to + initialize internal data structures; +- run: these interfaces are used to make the AP compute its output fields from + its input fields; +- finalization: these interfaces are used to perform any clean up operation + (e.g., release files) before the AP is destroyed. + +Among the above, the initialization sequence is the most complex, and consists +of several steps: + +- The AD creates the APG corresponding to the whole atmosphere. As mentioned + above, this phase will make use of a factory, which allows the AD to be + agnostic to what is actually in the group. All AP's can start performing any + initialization work that they can, but at this point they are limited to use + only an MPI communicator as well as a list of runtime parameters (which were + previously read from an input file). +- The AD passes a `GridsManager` to the AP's, so that they can get information + about the grids they need. At this point, all AP's have all the information + they need to establish the layout of the input and output fields they need, + and can store a list of these "requests" +- After creating all fields (based on AP's requests), the AD passes a copy of + each input and output field to the AP's. These fields will be divided in + "required" and "computed", which differ in that the former are only passed + to the AP's as 'read-only' fields (see the [field](field.md) + documentation for more details) +- The AP's are queried for how much scratch memory they may need at run time. + After all AP's communicate their needs, the AD will provide a pointer to + scratch memory to the AP's. This is memory that can be used to initialize + temporary views/fields or other internal data structures. All AP's are given + the same pointer, which means no data persistence should be expected at run + time between one timestep and the next. +- The AD calls the 'initialize' method on each AP. At this point, all fields + are set, and AP's can complete any remaining initialization task + +While the base AP class provides an (empty) implementation for some methods, in +case derived classes do not need a feature, some methods are purely virtual, +and concrete classes will have to override them. Looking at existing concrete +AP implementations is a good way to have a first idea of what a new AP class +needs to implement. Here, we show go over the possible implementation of these +methods in a hypothetical AP class. The header file may look something like +this + +```c++ +#include + +class MyProcess : public AtmosphereProcess +{ +public: + using gm_ptr = std::shared_ptr; + + MyProcess(const ekat::Comm& comm, const ekat::ParameterList& pl); + + std::string name () const override { return "my_fancy_process"; } + void set_grids (const gm_ptr& grids_manager) override; + size_t requested_buffer_size_in_bytes () const override; + void init_buffers (const ATMBufferManager& buffer_manager) override; +protected: + + void initialize_impl (const RunType run_type) override; + void run_impl (const double dt) override; + void finalize_impl () override; + + using view_1d = typename KokkosTypes::view_1d; + using view_2d = typename KokkosTypes::view_2d; + + view_1d m_temp1; + view_2d m_temp2; + + int m_ncols; + int m_nlevs; + bool m_has_blah; +}; +``` + +A few comments: + +- we added two views to the class, which are meant to be used to store + intermediate results during calculations at runtime; +- there are other methods that the class can override (such as additional + operations when the AD sets a field in the AP), but most AP's only need to + override only these; +- we strongly encourage to add the keyword `override` when overriding a method; + in case of small typos (e.g., missing a `&` or a `const`, the compiler will + be erroring out, since the signature will not match any virtual method in the + base class; +- `finalize_impl` is often empty; unless the AP is managing external resources, + everything should be correctly released during destruction; +- the two methods for buffers can be omitted if the AP does not need any + scratch memory (and the default implementation from the base class will be + used). + +Here is a possible implementation of the methods, with some inline comments to +explain + +```c++ +MyProcess::MyProcess (const ekat::Comm& comm, const ekat::ParameterList& pl) + : AtmosphereProcess(comm,pl) +{ + // The base class copies pl into protected member m_params + m_has_blah = m_params.get("enable_blah"); +} + +void MyProcess::set_grids (const std::shared_ptr& gm) +{ + using namespace ekat::units; + const auto nondim = Units::nondimensional(); + + auto grid = gm->get_grid("Physics"); + m_ncols = grid->get_num_local_dofs(); + m_nlevs = grid->get_num_vertical_levels(); + + // In these names, 2d refers to "horizontal only", while 3d to "horiz+vert". + // But the grid stores dofs linearly, so there is only one array dimension + FieldLayout scalar2d = grid->get_2d_scalar_layout(); + FieldLayout vector3d = grid->get_3d_vector_layout(true,2); + + // Declare fields needed: + // - Required: 'input' (read-only) + // - Computed: 'output' + // - Updated: 'input'+'output' + // Tell the AD we need 'velocity' to accommodate a Pack scalar type + add_field("coeff_2d",scalar2d,nondim,grid->name); + add_field("velocity",vector3d,m/s,grid->name,SCREAM_PACK_SIZE); +} + +size_t MyProcess::requested_buffer_size_in_bytes () +{ + // We use temp2 only if the blah feature is on + return m_ncols + (m_has_blah ? m_ncols*m_nlev : 0); +} + +void MyProcess::init_buffers (const ATMBufferManager& buffer_manager) +{ + auto mem = reinterpret_cast(buffer_manager.get_memory()); + + m_temp1 = view_1d(mem,m_ncols); + mem += m_ncols; + + if (m_has_blah) { + m_temp2 = view_2d(mem,m_ncols,m_nlevs); + mem += m_ncols*m_nlevs; + } + + // Make sure we use exactly the mem we said we would + size_t used_mem = (mem - buffer_manager.get_memory())*sizeof(Real); + EKAT_REQUIRE_MSG(used_mem==requested_buffer_size_in_bytes(), + "Error! Used memory != requested memory for MyProcess." + " used memory: " + std::to_string(used_mem) + "\n" + " requested: " + std::to_string(requested_buffer_size_in_bytes()) + "\n"); +} + +void MyProcess::initialize_impl(const RunType run_type) +{ + // Can complete any initialization of the background pkg + my_process_pkg_init(m_has_blah); +} + +void MyProcess:run_impl (const double dt) +{ + using Policy = typename KokkosTypes::TeamPolicy + using Member = typename KokkosTypes::MemberType + using PackT = ekat::Pack; + + // Create team policy + Policy policy(m_ncols,m_nlevs,1); + + // Create local copies of class members (or else use KOKKOS_CLASS_LAMBDA) + auto temp1 = m_temp1; + auto temp2 = m_temp2; + auto do_blah = m_has_blah; + + // Get views from fields. We + auto coeff2d = get_field_in("coeff_2d").get_view(); + auto velocity = get_field_out("velocity").get_view(); + + // Since we process velocity with a Pack scalar type, find out how many packs + // we have in each column + auto nlevs_packs = ekat::PackInfo::num_packs(m_nlevs); + + // Call some function in the background pkg + do_some_work (coeff_2d,velocity,temp1,temp2,do_blah); + + // Do some more work here + auto work = KOKKOS_LAMBDA (const Member& team) { + int icol = team.league_rank(); + auto col_work = [&](const int ilev) { + velocity(icol,ilev) *= coeff_2d; + }; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team,nlevs_packs),col_work); + }; + Kokkos::parallel_for(policy,work); + Kokkos::fence(); +} + +void MyProcess::finalize_impl () +{ + // If the background package needs to cleanup something, do it now + my_process_pkg_cleanup(); +} +``` diff --git a/components/eamxx/docs/developer/source_tree.md b/components/eamxx/docs/developer/source_tree.md index 15c018cc8858..ed8270db635e 100644 --- a/components/eamxx/docs/developer/source_tree.md +++ b/components/eamxx/docs/developer/source_tree.md @@ -56,4 +56,3 @@ You'll also see some other files in the `src/` directory itself, such as + `scream_config.h.in`: A template for generating a C++ header file with EAMxx configuration information. - diff --git a/components/eamxx/docs/developer/standalone_testing.md b/components/eamxx/docs/developer/standalone_testing.md index 63ea01fc612e..633dcc34dc1d 100644 --- a/components/eamxx/docs/developer/standalone_testing.md +++ b/components/eamxx/docs/developer/standalone_testing.md @@ -21,13 +21,53 @@ configurations. We use several types of tests the [ctest](https://cmake.org/cmake/help/latest/manual/ctest.1.html) command. We also support a `test-all-scream` configuration that runs all of the -standalone tests for an EAMxx configuration. +standalone tests for an EAMxx configuration. Note, your current machine +must be known to EAMxx before `test-all-scream` will work. A machine can +be made known to EAMxx by editing the eamxx/scripts/machines_specs.py files. +There are some instructions on what to do at the top of this file. + +`test-all-scream` has a good help dump + +```shell +cd $scream_repo/components/eamxx +./scripts/test-all-scream -h +``` + +If you are unsure of the cmake configuration for you development cycle, one +trick you can use is to run `test-all-scream` for the `dbg` test and just +copy the cmake command it prints (then ctrl-C the process). + +```shell +cd $scream_repo/components/eamxx +./scripts/test-all-scream -t dbg -m $machine +# wait for a few seconds* +# Ctrl-C * +# Copy the contents of DCMAKE_COMMAND that was passed to ctest * +# Add "cmake" to beginning of contents and path to eamxx at the end. * +``` + +Considerations for using `test-all-scream`: + +* Your machine must be known to our scripts, see above. +* If you try to run commands by-hand (outside of test-all-scream; + cmake, make, ctest, etc), you'll need to remember to + load the scream-env into your shell, which can be done like this: + `cd eamxx/scripts; eval $(./scripts/scream-env-cmd $machine)` +* test-all-scream expects to be run from a compute node if you + are on a batch machine. +* You'll need to think about your baseline situation, as many of our + tests rely on pre-existing baselines. The -b option controls the baseline + location and can have the following values: + * AUTO: A common public baseline area shared by all developers + * LOCAL: A private baseline area for the current developer in the current repo + * $path: A specific arbitrary path + * None: If there is no -b at all, no baseline testing will be done ## Running EAMxx's Tests with CTest Before running the tests, generate a baseline file: -``` +```shell cd $RUN_ROOT_DIR make baseline ``` @@ -39,7 +79,7 @@ path has been provided. To run all of SCREAM's tests, make sure you're in `$RUN_ROOT_DIR` and type -``` +```shell ctest -VV ``` @@ -48,7 +88,7 @@ This runs everything and reports results in an extra-verbose (`-VV`) manner. You can also run subsets of the SCREAM tests. For example, to run only the P3 regression tests (again, from the `$RUN_ROOT_DIR` directory), use -``` +```shell ctest -R p3_regression ``` @@ -58,13 +98,13 @@ We can create groupings of tests by using **labels**. For example, we have a `driver` label that runs tests for SCREAM's standalone driver. You can see a list of available labels by typing -``` +```shell ctest --print-labels ``` To see which tests are associated with a given label (e.g. `driver`), use -``` +```shell ctest -L driver -N ``` @@ -81,4 +121,3 @@ on the C++/Kokkos implementation, you can invoke any new tests to the function If the reference Fortran implementation changes enough that a new baseline file is required, make sure to let other SCREAM team members know, in order to minimize disruptions. - diff --git a/components/eamxx/docs/developer/style_guide.md b/components/eamxx/docs/developer/style_guide.md index f43678330099..4f6f340cb66e 100644 --- a/components/eamxx/docs/developer/style_guide.md +++ b/components/eamxx/docs/developer/style_guide.md @@ -7,4 +7,3 @@ Here's our style guide. Let the holy wars begin! ## Functions and Methods ## Variables - diff --git a/components/eamxx/docs/index.md b/components/eamxx/docs/index.md index d243f3c27560..0f78e45dbd46 100644 --- a/components/eamxx/docs/index.md +++ b/components/eamxx/docs/index.md @@ -1,6 +1,6 @@ # The E3SM Atmosphere Model in C++ (EAMxx) -EAMxx +##EAMxx EAMxx is almost completely different in all ways from the atmosphere model used for E3SM versions 1-3. diff --git a/components/eamxx/docs/old/physics/shoc/shoc_doc.tex b/components/eamxx/docs/old/physics/shoc/shoc_doc.tex index 66ae8b7baacc..21adca9ec317 100644 --- a/components/eamxx/docs/old/physics/shoc/shoc_doc.tex +++ b/components/eamxx/docs/old/physics/shoc/shoc_doc.tex @@ -217,7 +217,7 @@ \subsubsection{Eddy Diffusivities} \end{equation} % -Where $\lambda_{min} = 0.001$, $\lambda_{slope}$ = 0.35, and $N_{low}$ = 0.037. Here, $\lambda_{slope}$ is an adjustable parameter. $\lambda_{0}$ has a minimum threshold of 0.001 and a maximum threshold of 0.04. +Where $\lambda_{min} = 0.001$, $\lambda_{slope}$ = 0.35, and $N_{low}$ = 0.037. Here, $\lambda_{slope}$ is an adjustable parameter. $\lambda_{0}$ has a minimum threshold of 0.001 and a maximum threshold of 0.08. \paragraph{Stable Boundary Layer} diff --git a/components/eamxx/docs/user/dp_eamxx.md b/components/eamxx/docs/user/dp_eamxx.md new file mode 100644 index 000000000000..eeb71ee66444 --- /dev/null +++ b/components/eamxx/docs/user/dp_eamxx.md @@ -0,0 +1,3 @@ +# Doubly Periodic (DP) EAMxx + +To run the DP configuration of EAMxx (DP-EAMxx) please refer to the official [DP resource page](https://github.com/E3SM-Project/scmlib/wiki/Doubly-Periodic-SCREAM-Home). At this location you will find full documentation of case descriptions and access to run scripts. Using these scripts, you should be able to get DP-EAMxx up and running in a matter of minutes on any machine that EAMxx currently runs on (CPU or GPU). diff --git a/components/eamxx/docs/user/index.md b/components/eamxx/docs/user/index.md index 2e24e9fa736a..11eb659f3fd4 100644 --- a/components/eamxx/docs/user/index.md +++ b/components/eamxx/docs/user/index.md @@ -1,4 +1,4 @@ -# SCREAM User Guide +# EAMxx User Guide This section contains documentation on how to create, setup, and run CIME cases with EAMxx as the atmosphere component. It is assumed that the reader has a familiarity with [CIME case @@ -8,3 +8,14 @@ that the user knows how to create a case, and what the `case.setup`, `case.build This user guide is still under construction. In the meantime, in case you can't find the information you need, you may visit our public confluence [EAMxx user guide](https://acme-climate.atlassian.net/wiki/spaces/DOC/pages/3858890786/EAMxx+User+s+Guide). + +### [EAMxx case basics](eamxx_cases.md) +### [Model input](model_input.md) +### [Model output](model_output.md) +### [Nudging](nudging.md) +### [Extra radiation calls](clean_clear_sky.md) +### [COSP](cosp.md) +### [Regionally Refined EAMxx](rrm_eamxx.md) +### [Doubly Periodic EAMxx](dp_eamxx.md) +### [PySCREAM](pyscream.md) + diff --git a/components/eamxx/docs/user/pyscream.md b/components/eamxx/docs/user/pyscream.md new file mode 100644 index 000000000000..e49993132ac0 --- /dev/null +++ b/components/eamxx/docs/user/pyscream.md @@ -0,0 +1,73 @@ +# PySCREAM + +PySCREAM is currently under heavy development and may contain some +rough edges. If you encounter any issues, please report them on the +team on +[github discussions](https://github.com/E3SM-Project/E3SM/labels/eamxx). +Likewise, if you have questions or would like to request features, +please post them on the +[github discussions](https://github.com/E3SM-Project/E3SM/labels/eamxx). + +## Quick Start + +For now, the only way to use pyscream is to either build it on your own +or use our prebuilt conda binaries. We prefer for you to use the latter. +In a conda environment, please use the following command to install it: + +```bash +conda install -c mahf708 pyscream=0.0.2 +``` + +It is recommended to use the latest version of pyscream, wich is +currently 0.0.2. As you can see, it is a young package with a lot of +potential. We do not guarantee that the API will remain stable, but we +will try to document any changes as frequently as we could. + +## Examples + +We provide an example to demo calling the radiation process (RRTMGP). +More examples are on the way. If you'd like to add your example, +please feel free to submit a PR. + +### RRTMGP + +```python +from mpi4py import MPI +import pyscream + +pyscream.init() + +dt = 1800 +t0_str = "2020-10-10-00000" + +ic_file = "/lcrc/group/e3sm/public_html/inputdata/atm/scream/init/screami_unit_tests_ne2np4L72_20220822.nc" +ncols = 218 +nlevs = 72 +pyscream.create_grids_manager(ncols,nlevs, ic_file) + +rad_dict = { + "column_chunk_size": 123, + "active_gases": ["h2o", "co2", "o3", "n2o", "co" , "ch4", "o2", "n2"], + "orbital_year": 1990, + "log_level": "info", + "do_aerosol_rad": False, + "rrtmgp_coefficients_file_sw": "/lcrc/group/e3sm/data/inputdata/atm/scream/init/rrtmgp-data-sw-g112-210809.nc", + "rrtmgp_coefficients_file_lw": "/lcrc/group/e3sm/data/inputdata/atm/scream/init/rrtmgp-data-lw-g128-210809.nc", + "rrtmgp_cloud_optics_file_sw": "/lcrc/group/e3sm/data/inputdata/atm/scream/init/rrtmgp-cloud-optics-coeffs-sw.nc", + "rrtmgp_cloud_optics_file_lw": "/lcrc/group/e3sm/data/inputdata/atm/scream/init/rrtmgp-cloud-optics-coeffs-lw.nc", +} + +rad = pyscream.AtmProc(rad_dict, 'RRTMGP') +rad.read_ic(ic_file) +rad.initialize(t0_str) + +t = rad.get_field("T_mid") +tm = t.get() + +print(tm[5,5], flush=True) + +rad.run(dt) +rad.run(dt) + +print(tm[5,5], flush=True) +``` diff --git a/components/eamxx/docs/user/rrm_eamxx.md b/components/eamxx/docs/user/rrm_eamxx.md new file mode 100644 index 000000000000..f0de14b3c09f --- /dev/null +++ b/components/eamxx/docs/user/rrm_eamxx.md @@ -0,0 +1,29 @@ +# Running EAMxx with a Regionally Refined Mesh (RRM) + +Running EAMxx with a RRM allows you run a select region of the globe at high resolution (i.e. 3 km) with the remainder of the globe at a lower resolution (i.e. 25 or 100 km). This document will point you to the steps required and resources available to assist in developing and running a new RRM. + +## Choose Your RRM + +What region of the globe do you want to refine? Your first step should be to check [library of RRM grids/cases](https://acme-climate.atlassian.net/wiki/spaces/DOC/pages/3690397775/Library+of+Regionally-Refined+Model+RRM+Grids) that have already been developed to potentially avoid duplicate work. If you found a RRM that suits your needs, you can skip the next step ("Generate Your RRM"). + +## Generate Your RRM + +Please refer to the offical [E3SM guide for developing new atmosphere grids](https://acme-climate.atlassian.net/wiki/spaces/DOC/pages/872579110/Running+E3SM+on+New+Atmosphere+Grids), which provides detailed guidance for developing your RRM. + +After you have made all the necessary files for your RRM, you will need to configure your code branch so that it knows about your new grid. The steps required to do this are documented at the top of the [library of RRM grids/cases page](https://acme-climate.atlassian.net/wiki/spaces/DOC/pages/3690397775/Library+of+Regionally-Refined+Model+RRM+Grids). + +## Make Your Initial Condition File + +The easiest way to generate an initial condition is to use the [HICCUP tool](https://github.com/E3SM-Project/HICCUP), which is a set of flexible and robust python routines to streamline the task of generating a new atmospheric initial condition for E3SM. Otherwise, please see the step-by-step instuctions if you prefer to [manually generate your initial condition](https://acme-climate.atlassian.net/wiki/spaces/DOC/pages/1002373272/Generate+atm+initial+condition+from+analysis+data). + +## Assemble Nudging Data (Optional) + +If you wish to nudge your simulation, assemble your nudging data in the format required by EAMxx. Please refer to the [nudging documentation](nudging.md). + +In the event that you only want to nudge a portion of your domain, then you will need to generate a nudging weights file. A common use case for this is when you want the high-resolution region to remain free-running (unnudged) while nudging the coarse domain towards reanalysis or model data. Please use [this script](https://github.com/E3SM-Project/eamxx-scripts/blob/master/run_scripts/RRM_example_scripts/SCREAMv1_create_nudging_weights.py) as an example of how to generate your nudging weights file. + +## Run your RRM + +Congratulations, you are now ready to run your EAMxx RRM. If you are running your RRM in free running mode (not using any nudging) then you simply need to modify an existing EAMxx script and change the resolution to match the one you created for your RRM. + +If you are using nudging, then please see this [example script of how to run a nudged EAMxx RRM run](https://github.com/E3SM-Project/eamxx-scripts/blob/master/run_scripts/RRM_example_scripts/SCREAMv1-nudging.CAx32v1pg2.pm-gpu.template.sh). This example script uses the [California 3-km RRM](https://gmd.copernicus.org/articles/17/3687/2024/), which is on the master branch. diff --git a/components/eamxx/mkdocs.yaml b/components/eamxx/mkdocs.yml similarity index 93% rename from components/eamxx/mkdocs.yaml rename to components/eamxx/mkdocs.yml index 0ed6cfef9184..b1f5cace0a5c 100644 --- a/components/eamxx/mkdocs.yaml +++ b/components/eamxx/mkdocs.yml @@ -10,6 +10,9 @@ nav: - 'Nudging': 'user/nudging.md' - 'Extra radiation calls': 'user/clean_clear_sky.md' - 'COSP': 'user/cosp.md' + - 'Regionally Refined EAMxx': 'user/rrm_eamxx.md' + - 'Doubly Periodic EAMxx': 'user/dp_eamxx.md' + - 'PySCREAM': 'user/pyscream.md' - 'Developer Guide': - 'Overview': 'developer/index.md' - 'Installation': 'common/installation.md' diff --git a/components/eamxx/scripts/README.md b/components/eamxx/scripts/README.md index 0bcb8d79e6a1..bcff04a11cd2 100644 --- a/components/eamxx/scripts/README.md +++ b/components/eamxx/scripts/README.md @@ -80,10 +80,10 @@ This tool is not used in our core testing tools and goes through long periods of (IE there are no active ongoing porting efforts). It is likely this tool will not work exactly as expected if it has not been run in a while or is being used on a package on which it has not been used before. -## cf-xml-to-yaml +## query-cf-database/cf-xml-to-yaml Given an XML file containing the CF conventions for standardized field names -(https://cfconventions.org/standard-names.html), this tool generates a YAML +(which can be found [here](https://cfconventions.org/standard-names.html)), this tool generates a YAML file with the same information. This tool is not used in our core testing tools, but it is extremely simple and not coupled to anything else in the repo, diff --git a/components/eamxx/scripts/atmchange b/components/eamxx/scripts/atmchange index eb8de06a6931..6303cad6ca6c 100755 --- a/components/eamxx/scripts/atmchange +++ b/components/eamxx/scripts/atmchange @@ -15,7 +15,7 @@ sys.path.append(os.path.dirname(os.path.realpath(__file__))) from eamxx_buildnml_impl import check_value, is_array_type from eamxx_buildnml import create_raw_xml_file from atm_manip import atm_config_chg_impl, buffer_changes, reset_buffer, get_xml_nodes, parse_change -from utils import run_cmd_no_fail, expect +from utils import run_cmd_no_fail, expect, GoodFormatter # Add path to cime _CIMEROOT = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..","..","..","cime") @@ -110,7 +110,7 @@ OR > {0} foo=hi bar+=there """.format(pathlib.Path(args[0]).name), description=description, - formatter_class=argparse.ArgumentDefaultsHelpFormatter + formatter_class=GoodFormatter ) parser.add_argument( diff --git a/components/eamxx/scripts/atmquery b/components/eamxx/scripts/atmquery index 1dbfbe5f5795..2ca2692ccc88 100755 --- a/components/eamxx/scripts/atmquery +++ b/components/eamxx/scripts/atmquery @@ -14,7 +14,7 @@ sys.path.append(os.path.dirname(os.path.realpath(__file__))) from eamxx_buildnml_impl import check_value from atm_manip import atm_query_impl -from utils import expect +from utils import expect, GoodFormatter ############################################################################### def atm_query(variables,listall=False,full=False,value=False, \ @@ -55,7 +55,7 @@ OR """.format(pathlib.Path(args[0]).name), description=description, - formatter_class=argparse.ArgumentDefaultsHelpFormatter + formatter_class=GoodFormatter ) parser.add_argument( diff --git a/components/eamxx/scripts/check-hashes-ers b/components/eamxx/scripts/check-hashes-ers index 6d9da4b2f98e..4bcfab36c146 100755 --- a/components/eamxx/scripts/check-hashes-ers +++ b/components/eamxx/scripts/check-hashes-ers @@ -10,7 +10,7 @@ hash output after a test has run. import sys, re, glob, pathlib, argparse -from utils import run_cmd_no_fail, expect +from utils import run_cmd_no_fail, expect, GoodFormatter ############################################################################### def parse_command_line(args, description): @@ -25,7 +25,7 @@ OR > {0} /my/case/dir """.format(pathlib.Path(args[0]).name), description=description, - formatter_class=argparse.ArgumentDefaultsHelpFormatter + formatter_class=GoodFormatter ) parser.add_argument( diff --git a/components/eamxx/scripts/check-input b/components/eamxx/scripts/check-input index 4bac4af8b9bf..a4d503311165 100755 --- a/components/eamxx/scripts/check-input +++ b/components/eamxx/scripts/check-input @@ -4,7 +4,7 @@ Download SCREAM input files using CIME """ -from utils import check_minimum_python_version +from utils import check_minimum_python_version, GoodFormatter check_minimum_python_version(3, 4) import argparse, sys, pathlib @@ -24,7 +24,7 @@ OR > {0} /my/input/root my/file """.format(pathlib.Path(args[0]).name), description=description, - formatter_class=argparse.ArgumentDefaultsHelpFormatter + formatter_class=GoodFormatter ) parser.add_argument("input_root", help="The root input area (called DIN_LOC_ROOT in CIME)") diff --git a/components/eamxx/scripts/check-tendencies b/components/eamxx/scripts/check-tendencies index c044995f5d2a..4a3b42aa27dc 100755 --- a/components/eamxx/scripts/check-tendencies +++ b/components/eamxx/scripts/check-tendencies @@ -10,7 +10,7 @@ (other dims omitted for brevity). """ -from utils import check_minimum_python_version +from utils import check_minimum_python_version, GoodFormatter check_minimum_python_version(3, 4) import argparse, sys, pathlib @@ -24,7 +24,7 @@ def parse_command_line(args, description): usage=" {0} -f -v -t " .format(pathlib.Path(args[0]).name), description=description, - formatter_class=argparse.RawDescriptionHelpFormatter + formatter_class=GoodFormatter ) # The name of the nc files where to grab data from diff --git a/components/eamxx/scripts/check_input.py b/components/eamxx/scripts/check_input.py index f17974f26563..e5fa87ff1b92 100644 --- a/components/eamxx/scripts/check_input.py +++ b/components/eamxx/scripts/check_input.py @@ -9,6 +9,7 @@ from CIME.case.check_input_data import _download_if_in_repo from CIME.utils import expect from CIME.XML.inputdata import Inputdata +from CIME.Servers import has_svn, has_gftp, has_ftp, has_wget ############################################################################### def download_file(input_root, the_file): @@ -19,16 +20,16 @@ def download_file(input_root, the_file): while not success and protocol is not None: protocol, address, user, passwd, _, ic_filepath, _ = inputdata.get_next_server() if protocol is not None: - if protocol == "svn": + if protocol == "svn" and has_svn: from CIME.Servers import SVN server = SVN(address, user, passwd) - elif protocol == "gftp": + elif protocol == "gftp" and has_gftp: from CIME.Servers import GridFTP server = GridFTP(address, user, passwd) - elif protocol == "ftp": + elif protocol == "ftp" and has_ftp: from CIME.Servers import FTP server = FTP.ftp_login(address, user, passwd) - elif protocol == "wget": + elif protocol == "wget" and has_wget: from CIME.Servers import WGET server = WGET.wget_login(address, user, passwd) else: diff --git a/components/eamxx/scripts/cime-nml-tests b/components/eamxx/scripts/cime-nml-tests index d5ed8e14f2e0..c76d0113207f 100755 --- a/components/eamxx/scripts/cime-nml-tests +++ b/components/eamxx/scripts/cime-nml-tests @@ -6,7 +6,7 @@ namelist-related infrastructure. """ from utils import check_minimum_python_version, expect, ensure_pylint, get_timestamp, \ - run_cmd_assert_result, run_cmd_no_fail, run_cmd + run_cmd_assert_result, run_cmd_no_fail, run_cmd, GoodFormatter check_minimum_python_version(3, 6) @@ -256,7 +256,7 @@ class TestBuildnml(unittest.TestCase): def_mach_comp = \ run_cmd_assert_result(self, "../CIME/Tools/list_e3sm_tests cime_tiny", from_dir=CIME_SCRIPTS_DIR).splitlines()[-2].split(".")[-1] - case = self._create_test(f"SMS.ne30_ne30.F2010-SCREAMv1.{def_mach_comp}.scream-scream_example_testmod_atmchange --no-build") + case = self._create_test(f"SMS.ne30_ne30.F2010-SCREAMv1.{def_mach_comp}.eamxx-scream_example_testmod_atmchange --no-build") # Check that the value match what's in the testmod out = run_cmd_no_fail("./atmquery --value cubed_sphere_map", from_dir=case) @@ -410,7 +410,7 @@ OR parser = argparse.ArgumentParser( usage=help_str, description=desc, - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + formatter_class=GoodFormatter) parser.add_argument("-m", "--machine", help="Provide machine name. This is required for full (not dry) runs") diff --git a/components/eamxx/scripts/compare-nc-files b/components/eamxx/scripts/compare-nc-files index 9ca9675545ef..f8857b530ce5 100755 --- a/components/eamxx/scripts/compare-nc-files +++ b/components/eamxx/scripts/compare-nc-files @@ -5,7 +5,7 @@ Populates a netcdf file adding requested variables, either importing them from another file, or by computing them as function of other existing ones. """ -from utils import check_minimum_python_version +from utils import check_minimum_python_version, GoodFormatter check_minimum_python_version(3, 4) import argparse, sys, pathlib @@ -36,7 +36,7 @@ OR """.format(pathlib.Path(args[0]).name), description=description, - formatter_class=argparse.ArgumentDefaultsHelpFormatter + formatter_class=GoodFormatter ) # The name of the nc files where to grab data from diff --git a/components/eamxx/scripts/eamxx-params-docs-autogen b/components/eamxx/scripts/eamxx-params-docs-autogen index 9bea5242d5a9..90e19ed59d6a 100755 --- a/components/eamxx/scripts/eamxx-params-docs-autogen +++ b/components/eamxx/scripts/eamxx-params-docs-autogen @@ -10,7 +10,7 @@ a doc string and its type, as well as, if present, constraints and valid values. import argparse, sys, os, pathlib -from utils import _ensure_pylib_impl +from utils import _ensure_pylib_impl, GoodFormatter _ensure_pylib_impl("mdutils") @@ -29,7 +29,7 @@ def parse_command_line(args, description): usage="""{0} """.format(pathlib.Path(args[0]).name), description=description, - formatter_class=argparse.ArgumentDefaultsHelpFormatter + formatter_class=GoodFormatter ) return parser.parse_args(args[1:]) diff --git a/components/eamxx/scripts/gather-all-data b/components/eamxx/scripts/gather-all-data index be2153498dbc..f56bd416b20e 100755 --- a/components/eamxx/scripts/gather-all-data +++ b/components/eamxx/scripts/gather-all-data @@ -20,7 +20,7 @@ A number of options support automatic magic strings. See the help for individual options to see which magic strings are supported. """ -from utils import expect, check_minimum_python_version +from utils import expect, check_minimum_python_version, GoodFormatter from git_utils import get_current_commit check_minimum_python_version(3, 5) @@ -52,7 +52,7 @@ OR > {0} '$scream/scripts/perf-analysis ncol:8000 km1:128 km2:256 minthresh:0.001 repeat:10 --kokkos=$kokkos --test=lin-interp/li_ref --test=lin-interp/li_kokkos --test=lin-interp/li_vect -s ncol:2:128000' -m blake -o """.format(pathlib.Path(args[0]).name), description=description, - formatter_class=argparse.ArgumentDefaultsHelpFormatter + formatter_class=GoodFormatter ) parser.add_argument("run", help="What to run on the machine, cwd for the command will be root-dir. " diff --git a/components/eamxx/scripts/gather_all_data.py b/components/eamxx/scripts/gather_all_data.py index 9c9b1726c465..1881b304092d 100644 --- a/components/eamxx/scripts/gather_all_data.py +++ b/components/eamxx/scripts/gather_all_data.py @@ -2,8 +2,7 @@ import os, pathlib import concurrent.futures as threading3 -from machines_specs import get_mach_env_setup_command, get_mach_batch_command, \ - get_mach_cxx_compiler, get_mach_f90_compiler, get_mach_c_compiler +from machines_specs import get_mach_env_setup_command, get_machine ############################################################################### class GatherAllData(object): @@ -14,7 +13,7 @@ def __init__(self, run, commit, machines, scream_docs, local, kokkos, root_dir, ########################################################################### self._run = run self._commit = commit - self._machines = machines + self._machines = [get_machine(m) for m in machines] self._scream_docs = scream_docs self._local = local self._kokkos = kokkos @@ -27,16 +26,11 @@ def formulate_command(self, machine): # gather-all runs on login, not compute nodes, so we cannot rely on # probed values to be accurate, so we should not set CTEST_PARALLEL_LEVEL # based on these potentially wrong values. - env_setup = get_mach_env_setup_command(machine, ctest_j=-1) - - cxx_compiler = get_mach_cxx_compiler(machine) - f90_compiler = get_mach_f90_compiler(machine) - c_compiler = get_mach_c_compiler(machine) - batch = get_mach_batch_command(machine) + env_setup = get_mach_env_setup_command(machine.name, ctest_j=-1) env_setup_str = " && ".join(env_setup) - root_dir = pathlib.Path(str(self._root_dir).replace("$machine", machine)) + root_dir = pathlib.Path(str(self._root_dir).replace("$machine", machine.name)) # Compute locations of key repos if self._local: @@ -49,7 +43,7 @@ def formulate_command(self, machine): else: if self._scream_docs: scream_docs_repo = root_dir - scream_repo = pathlib.Path("~/scream-perf-{}/components/eamxx".format(machine)) + scream_repo = pathlib.Path(f"~/scream-perf-{machine.name}/components/eamxx") else: scream_docs_repo = None scream_repo = root_dir @@ -68,10 +62,10 @@ def formulate_command(self, machine): # Do magic replacements here local_cmd = local_cmd.\ replace("$kokkos", str(kokkos_loc)).\ - replace("$cxx_compiler", cxx_compiler).\ - replace("$f90_compiler", f90_compiler).\ - replace("$c_compiler", c_compiler).\ - replace("$machine", machine).\ + replace("$cxx_compiler", machine.cxx_compiler).\ + replace("$f90_compiler", machine.ftn_compiler).\ + replace("$c_compiler", machine.c_compiler).\ + replace("$machine", machine.name).\ replace("$scream_docs", str(scream_docs_repo)).\ replace("$scream", str(scream_repo)) @@ -85,7 +79,7 @@ def formulate_command(self, machine): repo_setup = "true" if (self._local) else "git fetch && git checkout {} && git submodule update --init --recursive".format(self._commit) cmd = "{setup}cd {repo} && {env_setup} && {repo_setup} && {batch} {local_cmd}".\ - format(setup=setup, repo=repo, env_setup=env_setup_str, repo_setup=repo_setup, batch=batch, local_cmd=local_cmd) + format(setup=setup, repo=repo, env_setup=env_setup_str, repo_setup=repo_setup, batch=machine.batch, local_cmd=local_cmd) return cmd @@ -93,20 +87,20 @@ def formulate_command(self, machine): def run_on_machine(self, machine): ########################################################################### cmd = self.formulate_command(machine) - print("Starting analysis on {} with cmd: {}".format(machine, cmd)) + print(f"Starting analysis on {machine.name} with cmd: {cmd}") if self._local: run_cmd_no_fail(cmd, arg_stdout=None, arg_stderr=None, verbose=True, dry_run=self._dry_run, exc_type=RuntimeError) else: output = "" # Making pylint happy try: - ssh_cmd = "ssh -o StrictHostKeyChecking=no {} '{}'".format(machine, cmd) + ssh_cmd = f"ssh -o StrictHostKeyChecking=no {machine.name} '{cmd}'" output = run_cmd_no_fail(ssh_cmd, dry_run=self._dry_run, exc_type=RuntimeError, combine_output=True) except RuntimeError as e: output = str(e) raise finally: - result_path = os.path.join("gather-all-results", self._commit, machine) + result_path = os.path.join("gather-all-results", self._commit, machine.name) if os.path.exists(result_path): old_path = result_path + ".old" while (os.path.exists(old_path)): @@ -117,7 +111,7 @@ def run_on_machine(self, machine): with open(result_path, "w", encoding="utf-8") as fd: fd.write(output) - print("Completed analysis on {}".format(machine)) + print(f"Completed analysis on {machine.name}") ########################################################################### def gather_all_data(self): @@ -135,7 +129,7 @@ def gather_all_data(self): try: future.result() except RuntimeError: - print('{} failed'.format(machine)) + print(f'{machine.name} failed') success = False else: @@ -143,7 +137,7 @@ def gather_all_data(self): try: self.run_on_machine(machine) except RuntimeError: - print('{} failed'.format(machine)) + print(f'{machine.name} failed') success = False return success diff --git a/components/eamxx/scripts/gen-boiler b/components/eamxx/scripts/gen-boiler index e1696d820a4f..47fb46d857eb 100755 --- a/components/eamxx/scripts/gen-boiler +++ b/components/eamxx/scripts/gen-boiler @@ -11,7 +11,7 @@ To test this tool, run: % python3 -m doctest gen_boiler.py """ -from utils import expect, check_minimum_python_version +from utils import expect, check_minimum_python_version, GoodFormatter from git_utils import get_git_toplevel_dir check_minimum_python_version(3, 5) @@ -42,7 +42,7 @@ OR > {0} foo_bar --piece=f90_f2c_bind --piece=f90_c2f_bind -o --physics=p3 """.format(pathlib.Path(args[0]).name), description=description, - formatter_class=argparse.RawTextHelpFormatter + formatter_class=GoodFormatter ) default_repo = get_git_toplevel_dir() diff --git a/components/eamxx/scripts/gen_boiler.py b/components/eamxx/scripts/gen_boiler.py index d45197deeeba..f22256516204 100644 --- a/components/eamxx/scripts/gen_boiler.py +++ b/components/eamxx/scripts/gen_boiler.py @@ -138,21 +138,21 @@ )), ("cxx_f2c_bind_decl" , ( - lambda phys, sub, gb: f"{phys}_functions_f90.hpp", + lambda phys, sub, gb: f"tests/infra/{phys}_test_data.hpp", lambda phys, sub, gb: expect_exists(phys, sub, gb, "cxx_f2c_bind_decl"), - lambda phys, sub, gb: get_cxx_close_block_regex(comment="end _f function decls"), # reqs special comment - lambda phys, sub, gb: get_cxx_function_begin_regex(sub + "_f"), # cxx_f decl + lambda phys, sub, gb: get_plain_comment_regex(comment="end _host function decls"), # reqs special comment + lambda phys, sub, gb: get_cxx_function_begin_regex(sub + "_host"), # cxx_host decl lambda phys, sub, gb: re.compile(r".*;\s*$"), # ; - lambda *x : "The f90 to cxx function declaration(_f)" + lambda *x : "The f90 to cxx function declaration(_host)" )), ("cxx_f2c_bind_impl" , ( - lambda phys, sub, gb: f"{phys}_functions_f90.cpp", + lambda phys, sub, gb: f"tests/infra/{phys}_test_data.cpp", lambda phys, sub, gb: expect_exists(phys, sub, gb, "cxx_f2c_bind_impl"), lambda phys, sub, gb: get_namespace_close_regex(phys), # insert at end of namespace - lambda phys, sub, gb: get_cxx_function_begin_regex(sub + "_f"), # cxx_f + lambda phys, sub, gb: get_cxx_function_begin_regex(sub + "_host"), # cxx_f lambda phys, sub, gb: get_cxx_close_block_regex(at_line_start=True), # terminating } - lambda *x : "The f90 to cxx function implementation(_f)" + lambda *x : "The f90 to cxx function implementation(_host)" )), ("cxx_func_decl", ( @@ -455,6 +455,12 @@ def get_cxx_struct_begin_regex(struct): struct_regex_str = fr"^\s*struct\s+{struct}([\W]|$)" return re.compile(struct_regex_str) +############################################################################### +def get_plain_comment_regex(comment): +############################################################################### + comment_regex_str = fr"^\s*//\s*{comment}" + return re.compile(comment_regex_str) + ############################################################################### def get_data_struct_name(sub): ############################################################################### @@ -1169,6 +1175,21 @@ def split_by_type(arg_data): return reals, ints, logicals +############################################################################### +def split_by_scalar_vs_view(arg_data): +############################################################################### + """ + Take arg data and split into two lists of names based on scalar/not-scalar: [scalars] [non-scalars] + """ + scalars, non_scalars = [], [] + for name, _, _, dims in arg_data: + if dims is not None: + non_scalars.append(name) + else: + scalars.append(name) + + return scalars, non_scalars + ############################################################################### def gen_cxx_data_args(physics, arg_data): ############################################################################### @@ -1441,6 +1462,30 @@ def check_existing_piece(lines, begin_regex, end_regex): return None if begin_idx is None else (begin_idx, end_idx+1) +############################################################################### +def get_data_by_name(arg_data, arg_name, data_idx): +############################################################################### + for name, a, b, c in arg_data: + if name == arg_name: + return [name, a, b, c][data_idx] + + expect(False, f"Name {arg_name} not found") + +############################################################################### +def get_rank_map(arg_data, arg_names): +############################################################################### + # Create map of rank -> [args] + rank_map = {} + for arg in arg_names: + dims = get_data_by_name(arg_data, arg, ARG_DIMS) + rank = len(dims) + if rank in rank_map: + rank_map[rank].append(arg) + else: + rank_map[rank] = [arg] + + return rank_map + # # Main classes # @@ -1505,10 +1550,10 @@ def _get_db(self, phys): db = parse_origin(origin_file.open(encoding="utf-8").read(), self._subs) self._db[phys].update(db) if self._verbose: - print("For physics {}, found:") + print(f"For physics {phys}, found:") for sub in self._subs: if sub in db: - print(" For subroutine {}, found args:") + print(f" For subroutine {sub}, found args:") for name, argtype, intent, dims in db[sub]: print(" name:{} type:{} intent:{} dims:({})".\ format(name, argtype, intent, ",".join(dims) if dims else "scalar")) @@ -1729,7 +1774,7 @@ def gen_cxx_f2c_bind_decl(self, phys, sub, force_arg_data=None): arg_data = force_arg_data if force_arg_data else self._get_arg_data(phys, sub) arg_decls = gen_arg_cxx_decls(arg_data) - return f"void {sub}_f({', '.join(arg_decls)});" + return f"void {sub}_host({', '.join(arg_decls)});" ########################################################################### def gen_cxx_f2c_bind_impl(self, phys, sub, force_arg_data=None): @@ -1809,8 +1854,166 @@ def gen_cxx_f2c_bind_impl(self, phys, sub, force_arg_data=None): impl = "" if has_arrays(arg_data): - # TODO - impl += " // TODO" + # + # Steps: + # 1) Set up typedefs + # 2) Sync to device + # 3) Unpack view array + # 4) Get nk_pack and policy + # 5) Get subviews + # 6) Call fn + # 7) Sync back to host + # + inputs, inouts, outputs = split_by_intent(arg_data) + reals, ints, bools = split_by_type(arg_data) + scalars, views = split_by_scalar_vs_view(arg_data) + all_inputs = inputs + inouts + all_outputs = inouts + outputs + + vreals = list(sorted(set(reals) & set(views))) + vints = list(sorted(set(ints) & set(views))) + vbools = list(sorted(set(bools) & set(views))) + + sreals = list(sorted(set(reals) & set(scalars))) + sints = list(sorted(set(ints) & set(scalars))) + sbools = list(sorted(set(bools) & set(scalars))) + + ivreals = list(sorted(set(vreals) & set(all_inputs))) + ivints = list(sorted(set(vints) & set(all_inputs))) + ivbools = list(sorted(set(vbools) & set(all_inputs))) + + ovreals = list(sorted(set(vreals) & set(all_outputs))) + ovints = list(sorted(set(vints) & set(all_outputs))) + ovbools = list(sorted(set(vbools) & set(all_outputs))) + + isreals = list(sorted(set(sreals) & set(all_inputs))) + isints = list(sorted(set(sints) & set(all_inputs))) + isbools = list(sorted(set(sbools) & set(all_inputs))) + + osreals = list(sorted(set(sreals) & set(all_outputs))) + osints = list(sorted(set(sints) & set(all_outputs))) + osbools = list(sorted(set(sbools) & set(all_outputs))) + + # + # 1) Set up typedefs + # + + # set up basics + impl += "#if 0\n" # There's no way to guarantee this code compiles + impl += " using SHF = Functions;\n" + impl += " using Scalar = typename SHF::Scalar;\n" + impl += " using Spack = typename SHF::Spack;\n" + impl += " using KT = typename SHF::KT;\n" + impl += " using ExeSpace = typename KT::ExeSpace;\n" + impl += " using MemberType = typename SHF::MemberType;\n\n" + + prefix_list = ["", "i", "b"] + type_list = ["Real", "Int", "bool"] + ktype_list = ["Spack", "Int", "bool"] + + # make necessary view types. Anything that's an array needs a view type + for view_group, prefix_char, typename in zip([vreals, vints, vbools], prefix_list, type_list): + if view_group: + rank_map = get_rank_map(arg_data, view_group) + for rank in rank_map: + if typename == "Real" and rank > 1: + # Probably this should be packed data + impl += f" using {prefix_char}view_{rank}d = typename SHF::view_{rank}d;\n" + else: + impl += f" using {prefix_char}view_{rank}d = typename SHF::view_{rank}d<{typename}>;\n" + + impl += "\n" + + # + # 2) Sync to device. Do ALL views, not just inputs + # + + for input_group, prefix_char, typename in zip([vreals, vints, vbools], prefix_list, type_list): + if input_group: + rank_map = get_rank_map(arg_data, input_group) + + for rank, arg_list in rank_map.items(): + impl += f" static constexpr Int {prefix_char}num_arrays_{rank} = {len(arg_list)};\n" + impl += f" std::vector<{prefix_char}view_{rank}d> {prefix_char}temp_d_{rank}({prefix_char}num_arrays_{rank});\n" + for rank_itr in range(rank): + dims = [get_data_by_name(arg_data, arg_name, ARG_DIMS)[rank_itr] for arg_name in arg_list] + impl += f" std::vector {prefix_char}dim_{rank}_{rank_itr}_sizes = {{{', '.join(dims)}}};\n" + dim_vectors = [f"{prefix_char}dim_{rank}_{rank_itr}_sizes" for rank_itr in range(rank)] + funcname = "ekat::host_to_device" if (typename == "Real" and rank > 1) else "ScreamDeepCopy::copy_to_device" + impl += f" {funcname}({{{', '.join(arg_list)}}}, {', '.join(dim_vectors)}, {prefix_char}temp_d_{rank});\n\n" + + # + # 3) Unpack view array + # + + for input_group, prefix_char, typename in zip([vreals, vints, vbools], prefix_list, type_list): + if input_group: + rank_map = get_rank_map(arg_data, input_group) + + for rank, arg_list in rank_map.items(): + impl += f" {prefix_char}view_{rank}d\n" + for idx, input_item in enumerate(arg_list): + impl += f" {input_item}_d({prefix_char}temp_d_{rank}[{idx}]){';' if idx == len(arg_list) - 1 else ','}\n" + impl += "\n" + + + # + # 4) Get nk_pack and policy, launch kernel + # + impl += " const Int nk_pack = ekat::npack(nlev);\n" + impl += " const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(shcol, nk_pack);\n" + impl += " Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) {\n" + impl += " const Int i = team.league_rank();\n\n" + + # + # 5) Get subviews + # + for view_group, prefix_char, typename in zip([vreals, vints, vbools], prefix_list, type_list): + if view_group: + for view_arg in view_group: + dims = get_data_by_name(arg_data, view_arg, ARG_DIMS) + if "shcol" in dims: + if len(dims) == 1: + impl += f" const Scalar {view_arg}_s = {view_arg}_d(i);\n" + else: + impl += f" const auto {view_arg}_s = ekat::subview({view_arg}_d, i);\n" + + impl += "\n" + + # + # 6) Call fn + # + kernel_arg_names = [] + for arg_name in arg_names: + if arg_name in views: + if "shcol" in dims: + kernel_arg_names.append(f"{arg_name}_s") + else: + kernel_arg_names.append(f"{arg_name}_d") + else: + kernel_arg_names.append(arg_name) + + impl += f" SHF::{sub}({', '.join(kernel_arg_names)});\n" + impl += " });\n" + + # + # 7) Sync back to host + # + for output_group, prefix_char, typename in zip([ovreals, ovints, ovbools], prefix_list, type_list): + if output_group: + rank_map = get_rank_map(arg_data, output_group) + + for rank, arg_list in rank_map.items(): + impl += f" std::vector<{prefix_char}view_{rank}d> {prefix_char}tempout_d_{rank}({prefix_char}num_arrays_{rank});\n" + for rank_itr in range(rank): + dims = [get_data_by_name(arg_data, arg_name, ARG_DIMS)[rank_itr] for arg_name in arg_list] + impl += f" std::vector {prefix_char}dim_{rank}_{rank_itr}_out_sizes = {{{', '.join(dims)}}};\n" + dim_vectors = [f"{prefix_char}dim_{rank}_{rank_itr}_out_sizes" for rank_itr in range(rank)] + funcname = "ekat::device_to_host" if (typename == "Real" and rank > 1) else "ScreamDeepCopy::copy_to_host" + impl += f" {funcname}({{{', '.join(arg_list)}}}, {', '.join(dim_vectors)}, {prefix_char}tempout_d_{rank});\n\n" + + impl += "#endif\n" + else: inputs, inouts, outputs = split_by_intent(arg_data) reals, ints, logicals = split_by_type(arg_data) diff --git a/components/eamxx/scripts/git-merge-ref b/components/eamxx/scripts/git-merge-ref index 24d767c03a54..363f0f6045fb 100755 --- a/components/eamxx/scripts/git-merge-ref +++ b/components/eamxx/scripts/git-merge-ref @@ -4,7 +4,7 @@ A command-line tool foro calling merge_git_ref """ -from utils import check_minimum_python_version +from utils import check_minimum_python_version, GoodFormatter check_minimum_python_version(3, 4) import argparse, sys, pathlib @@ -24,7 +24,7 @@ OR > {0} origin/master """.format(pathlib.Path(args[0]).name), description=description, - formatter_class=argparse.ArgumentDefaultsHelpFormatter + formatter_class=GoodFormatter ) parser.add_argument("git_ref", help="The git ref to merge") diff --git a/components/eamxx/scripts/jenkins/ghci-snl-cpu_setup b/components/eamxx/scripts/jenkins/ghci-snl-cpu_setup new file mode 100644 index 000000000000..029340f81e06 --- /dev/null +++ b/components/eamxx/scripts/jenkins/ghci-snl-cpu_setup @@ -0,0 +1 @@ +SCREAM_MACHINE=ghci-snl-cpu diff --git a/components/eamxx/scripts/jenkins/jenkins_common_impl.sh b/components/eamxx/scripts/jenkins/jenkins_common_impl.sh index 75330d612762..f1be10d4eda6 100755 --- a/components/eamxx/scripts/jenkins/jenkins_common_impl.sh +++ b/components/eamxx/scripts/jenkins/jenkins_common_impl.sh @@ -17,11 +17,11 @@ if [ ${#labels[@]} -gt 0 ]; then # We do have some labels. Look for some supported ones. for label in "${labels[@]}" do - if [ "$label" == "AT: Integrate Without Testing" ]; then + if [ "$label" == "AT: skip eamxx-all" ]; then skip_testing=1 - elif [ "$label" == "AT: Skip Stand-Alone Testing" ]; then + elif [ "$label" == "AT: skip eamxx-sa" ]; then test_SA=0 - elif [ "$label" == "AT: Skip v1 Testing" ]; then + elif [ "$label" == "AT: skip eamxx-v1" ]; then test_v1=0 elif [ "$label" == "scripts" ]; then test_scripts=1 @@ -137,7 +137,7 @@ if [ $skip_testing -eq 0 ]; then fi else - echo "SCREAM Stand-Alone tests were skipped, since the Github label 'AT: Skip Stand-Alone Testing' was found.\n" + echo "SCREAM Stand-Alone tests were skipped, since the Github label 'AT: skip eamxx-sa' was found.\n" fi # Run expensive tests on mappy only diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index af61c2bff5f9..dbb9363381ce 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -4,243 +4,435 @@ ensure_psutil() import psutil -CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","cime") - -# MACHINE -> (env_setup, # list of shell commands to set up scream-approved env -# compilers, # list of compilers [CXX, F90, C] -# batch submit prefix, # string shell commmand prefix -# pre-existing baselines root dir # string path) -MACHINE_METADATA = { - # NOTE: blake must use a different minor version of python than weaver - # this is so pip installs do not go into the same location for both machines. - # We have found some python modules are not shareable between these two machinse. - "blake" : (["module purge", "module load python/3.8.8/gcc/10.2.0", "module unload gcc/10.2.0", "module load openmpi/2.1.2 zlib git/2.9.4 cmake/3.19.3", - "export PATH=/ascldap/users/projects/e3sm/scream/libs/netcdf-fortran/install/blake/bin:$PATH", - "export PATH=/ascldap/users/projects/e3sm/scream/libs/netcdf-c/install/blake/bin:$PATH", - "export PATH=/ascldap/users/projects/e3sm/scream/libs/pnetcdf/install/blake/bin:$PATH", - "export INTEL_LICENSE_FILE=/home/projects/x86-64/intel/licenses/USE_SERVER.lic" # speeds up license lookup - ], - ["mpicxx","mpifort","mpicc"], - "salloc -N 1 srun -n1 --preserve-env", - "/home/projects/e3sm/scream/pr-autotester/master-baselines/blake/"), - "weaver" : (["source /etc/profile.d/modules.sh", "module purge", "module load cmake/3.25.1 git/2.39.1 python/3.10.8 py-netcdf4/1.5.8 gcc/11.3.0 cuda/11.8.0 openmpi netcdf-c netcdf-fortran parallel-netcdf netlib-lapack", "export HDF5_USE_FILE_LOCKING=FALSE", - ], - ["mpicxx","mpifort","mpicc"], - "bsub -I -q rhel8 -n 4 -gpu num=4", - "/home/projects/e3sm/scream/pr-autotester/master-baselines/weaver/"), - "mappy" : (["source /projects/sems/modulefiles/utils/sems-modules-init.sh", - "module purge", "module load sems-cmake/3.27.9 sems-git/2.42.0 sems-gcc/11.4.0 sems-openmpi-no-cuda/4.1.6 sems-netcdf-c/4.9.2 sems-netcdf-cxx/4.2 sems-netcdf-fortran/4.6.1 sems-parallel-netcdf/1.12.3 sems-openblas", - "export GATOR_INITIAL_MB=4000MB", - ], - ["mpicxx","mpifort","mpicc"], - "", - "/sems-data-store/ACME/baselines/scream/master-baselines"), - "lassen" : (["module --force purge", "module load git gcc/8.3.1 cuda/11.8.0 cmake/3.16.8 spectrum-mpi python/3.7.2", "export LLNL_USE_OMPI_VARS='y'", - "export PATH=/usr/workspace/e3sm/netcdf/bin:$PATH", - "export LD_LIBRARY_PATH=/usr/workspace/e3sm/netcdf/lib:$LD_LIBRARY_PATH", - ], - ["mpicxx","mpifort","mpicc"], - "bsub -Ip -qpdebug", - ""), - "ruby-intel" : (["module --force purge", "module use --append /usr/workspace/e3sm/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"], - ["mpicxx","mpifort","mpicc"], - "salloc --partition=pdebug", - ""), - "dane-intel" : (["module --force purge", "module use --append /usr/workspace/e3sm/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"], - ["mpicxx","mpifort","mpicc"], - "salloc --partition=pdebug", - ""), - "quartz-intel" : (["module --force purge", "module use --append /usr/workspace/e3sm/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"], - ["mpicxx","mpifort","mpicc"], - "salloc --partition=pdebug", - ""), - "quartz-gcc" : (["module --force purge", "module load StdEnv cmake/3.16.8 mkl/2019.0 gcc/8.3.1 netcdf-fortran/4.4.4 netcdf/4.4.1.1 pnetcdf/1.9.0 mvapich2/2.3"], - ["mpicxx","mpifort","mpicc"], - "salloc --partition=pdebug", - ""), - "syrah" : (["module --force purge", "module load StdEnv cmake/3.16.8 mkl/2019.0 intel/19.0.4 netcdf-fortran/4.4.4 netcdf/4.4.1.1 pnetcdf/1.9.0 mvapich2/2.3"], - ["mpicxx","mpifort","mpicc"], - "salloc --partition=pbatch --time=60", - ""), - "summit" : (["module purge", "module load cmake/3.18.4 gcc/9.3.0 spectrum-mpi/10.4.0.3-20210112 cuda/11.5.2 python/3.7-anaconda3 netcdf-c/4.8.0 netcdf-fortran/4.4.5 openblas/0.3.5","unset OMPI_CXX", "export OMPI_COMM_WORLD_RANK=0"], - ["mpicxx","mpifort","mpicc"], - "bsub -I -q batch -W 0:30 -P cli115 -nnodes 1", - "/gpfs/alpine/cli115/proj-shared/scream/master-baselines"), - "pm-cpu" : ([f"eval $({CIMEROOT}/CIME/Tools/get_case_env -c SMS.ne4pg2_ne4pg2.F2010-SCREAMv1.pm-cpu_gnu)"], - ["CC","ftn","cc"], - "salloc --time 00:30:00 --nodes=1 --constraint=cpu -q debug --account e3sm_g", - "/global/cfs/cdirs/e3sm/baselines/gnu/scream/pm-cpu"), - "pm-gpu" : ([f"eval $({CIMEROOT}/CIME/Tools/get_case_env -c SMS.ne4pg2_ne4pg2.F2010-SCREAMv1.pm-gpu_gnugpu)", "echo cuda=true"], - ["CC","ftn","cc"], - "salloc --time 02:00:00 --nodes=4 --constraint=gpu --gpus-per-node=4 --gpu-bind=none --exclusive -q regular --account e3sm_g", - "/global/cfs/cdirs/e3sm/baselines/gnugpu/scream/pm-gpu"), - "compy" : (["module purge", "module load cmake/3.19.6 gcc/8.1.0 mvapich2/2.3.1 python/3.7.3"], - ["mpicxx","mpifort","mpicc"], - "srun --time 02:00:00 --nodes=1 -p short --exclusive --account e3sm", - ""), - "chrysalis" : ([f"eval $({CIMEROOT}/CIME/Tools/get_case_env)", "export OMP_NUM_THREADS=1"], - ["mpic++","mpif90","mpicc"], - "srun --mpi=pmi2 -l -N 1 --kill-on-bad-exit --cpu_bind=cores", - "/lcrc/group/e3sm/baselines/chrys/intel/scream"), - "anlgce" : ([". /nfs/gce/software/spack/opt/spack/linux-ubuntu20.04-x86_64/gcc-9.3.0/lmod-8.3-6fjdtku/lmod/lmod/init/sh", "module purge", "module load autoconf/2.69-bmnwajj automake/1.16.3-r7w24o4 libtool/2.4.6-uh3mpsu m4/1.4.19-7fztfyz cmake/3.20.5-zyz2eld gcc/11.1.0-qsjmpcg zlib/1.2.11-p7dmb5p", "export LD_LIBRARY_PATH=/nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/mpich/4.0/gcc-11.1.0/lib:$LD_LIBRARY_PATH", "export PATH=/nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/mpich/4.0/gcc-11.1.0/bin:/nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-11.1.0/bin:$PATH", "export NetCDF_ROOT=/nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-11.1.0", "export PERL5LIB=/nfs/gce/projects/climate/software/perl5/lib/perl5"], - ["mpicxx","mpifort","mpicc"], - "", - ""), - "anlgce-ub22" : ([". /nfs/gce/software/custom/linux-ubuntu22.04-x86_64/spack/opt/spack/linux-ubuntu22.04-x86_64/gcc-11.2.0/lmod-8.5.6-hkjjxhp/lmod/lmod/init/sh", "module purge", "module load gcc/12.1.0", "export LD_LIBRARY_PATH=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/mpich/4.1.2/gcc-12.1.0/lib:$LD_LIBRARY_PATH", "export PATH=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/mpich/4.1.2/gcc-12.1.0/bin:/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-12.1.0/bin:$PATH", "export NetCDF_ROOT=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-12.1.0", "export PERL5LIB=/nfs/gce/projects/climate/software/perl5/lib/perl5"], - ["mpicxx","mpifort","mpicc"], - "", - ""), - "linux-generic" : ([],["mpicxx","mpifort","mpicc"],"", ""), - "linux-generic-debug" : ([],["mpicxx","mpifort","mpicc"],"", ""), - "linux-generic-serial" : ([],["mpicxx","mpifort","mpicc"],"", ""), - "ghci-snl-openmp" : ([], - ["mpicxx","mpifort","mpicc"], - "", - "/projects/e3sm/baselines/scream/master-baselines" - ), -} - -if pathlib.Path("~/.cime/scream_mach_specs.py").expanduser().is_file(): # pylint: disable=no-member - sys.path.append(str(pathlib.Path("~/.cime").expanduser())) - from scream_mach_specs import MACHINE_METADATA as LOCAL_MD # pylint: disable=import-error - if len(LOCAL_MD) == 4: - MACHINE_METADATA["local"] = LOCAL_MD - else: - print("WARNING! File '~/.cime/scream_mach_specs.py' was found, but the MACHINE_METADATA in there is badly formatted. Ignoring it.") +EAMXX_DIR = pathlib.Path(__file__).parent.parent +CIMEROOT = os.path.join(EAMXX_DIR,"..","..","cime") ############################################################################### -def get_all_supported_machines(): +def logical_cores_per_physical_core(): ############################################################################### - return MACHINE_METADATA.keys() + return psutil.cpu_count() // psutil.cpu_count(logical=False) ############################################################################### -def is_machine_supported(machine): +def get_cpu_ids_from_slurm_env_var(): ############################################################################### - return machine in MACHINE_METADATA.keys() + """ + Parse the SLURM_CPU_BIND_LIST, and use the hexadecimal value to determine + which CPUs on this node are assigned to the job + NOTE: user should check that the var is set BEFORE calling this function + """ + + cpu_bind_list = os.getenv('SLURM_CPU_BIND_LIST') + + expect (cpu_bind_list is not None, + "SLURM_CPU_BIND_LIST environment variable is not set. Check, before calling this function") + + # Remove the '0x' prefix and convert to an integer + mask_int = int(cpu_bind_list, 16) + + # Generate the list of CPU IDs + cpu_ids = [] + for i in range(mask_int.bit_length()): # Check each bit position + if mask_int & (1 << i): # Check if the i-th bit is set + cpu_ids.append(i) + + return cpu_ids ############################################################################### -def assert_machine_supported(machine): +def get_available_cpu_count(logical=True): ############################################################################### - expect(is_machine_supported(machine), - "Machine {} is not currently supported by scream testing system.\n" - " Note: you can also create a file `~/.cime/scream_mach_specs.py` with your local machine specs.".format(machine)) + """ + Get number of CPUs available to this process and its children. logical=True + will include hyperthreads, logical=False will return only physical cores + """ + if 'SLURM_CPU_BIND_LIST' in os.environ: + cpu_count = len(get_cpu_ids_from_slurm_env_var()) + else: + cpu_count = len(psutil.Process().cpu_affinity()) + + if not logical: + hyperthread_ratio = logical_cores_per_physical_core() + return int(cpu_count / hyperthread_ratio) + else: + return cpu_count ############################################################################### -def get_mach_env_setup_command(machine, ctest_j=None): +class Machine(object): ############################################################################### """ - ctest_j=None -> probe for hardware for testing resources - ctest_j=-1 -> Skip CTEST_PARALLEL_LEVEL + Parent class for objects describing a machine to use for EAMxx standalone testing. """ - assert_machine_supported(machine) + concrete = False + name = "" + mach_file = "" + env_setup = [] + gpu_arch = "none" + num_bld_res = -1 + num_run_res = -1 + batch = "" + cxx_compiler = "mpicxx" + c_compiler = "mpicc" + ftn_compiler = "mpifort" + baselines_dir = "" - mach_custom_env = MACHINE_METADATA[machine][0] - if ctest_j != -1: - ctest_j = get_mach_testing_resources(machine) if ctest_j is None else ctest_j - mach_custom_env.append("export CTEST_PARALLEL_LEVEL={}".format(ctest_j)) + @classmethod + def setup_base(cls,name,num_bld_res=-1,num_run_res=-1): + cls.name = name + cls.mach_file = pathlib.Path(EAMXX_DIR) / "cmake" / "machine-files" / (cls.name + ".cmake") + cls.num_bld_res = num_bld_res if num_bld_res > 0 else get_available_cpu_count() + cls.num_run_res = num_run_res if num_run_res > 0 else get_available_cpu_count() - if not is_cuda_machine(machine): - mach_custom_env.append("export OMP_PROC_BIND=spread") + @classmethod + def uses_gpu (cls): + return cls.gpu_arch!="none" - return mach_custom_env +############################################################################### +class Generic(Machine): +############################################################################### + concrete = True + + @classmethod + def setup(cls): + super().setup_base("linux-generic") + # Set baselines_dir to PWD/../ctest-build/baselines + cls.baselines_dir = os.path.join(os.path.dirname(__file__), '..', 'ctest-build', 'baselines') ############################################################################### -def get_mach_cxx_compiler(machine): +class CrayMachine(Machine): ############################################################################### - assert_machine_supported(machine) - return MACHINE_METADATA[machine][1][0] + @classmethod + def setup_cray(cls,name): + super().setup_base(name) + + cls.cxx_compiler = "CC" + cls.c_compiler = "cc" + cls.ftn_compiler = "ftn" ############################################################################### -def get_mach_f90_compiler(machine): +class PM(CrayMachine): ############################################################################### - assert_machine_supported(machine) - return MACHINE_METADATA[machine][1][1] + @classmethod + def setup_pm(cls,partition): + expect (partition in ['cpu', 'gpu'], "Unknown Perlmutter partition") + + super().setup_cray("pm-"+partition) + + compiler = "gnu" if partition=="cpu" else "gnugpu" + + cls.env_setup = [f"eval $({CIMEROOT}/CIME/Tools/get_case_env -c SMS.ne4pg2_ne4pg2.F2010-SCREAMv1.{cls.name}_{compiler})"] + cls.batch = f"salloc --account e3sm_g --constraint={partition}" + if partition=="cpu": + cls.batch += "--time 00:30:00 --nodes=1 -q debug" + else: + cls.batch += "--time 02:00:00 --nodes=4 --gpus-per-node=4 --gpu-bind=none --exclusive -q regular" + + cls.baselines_dir = f"/global/cfs/cdirs/e3sm/baselines/{compiler}/scream/{cls.name}" ############################################################################### -def get_mach_c_compiler(machine): +class PMCPU(PM): ############################################################################### - assert_machine_supported(machine) - return MACHINE_METADATA[machine][1][2] + concrete = True + @classmethod + def setup(cls): + super().setup_pm("cpu") ############################################################################### -def get_mach_batch_command(machine): +class PMGPU(PM): ############################################################################### - assert_machine_supported(machine) - return MACHINE_METADATA[machine][2] + concrete = True + @classmethod + def setup(cls): + super().setup_pm("gpu") + + cls.num_run_res = 4 # four gpus + cls.gpu_arch = "cuda" ############################################################################### -def get_mach_baseline_root_dir(machine): +class Chrysalis(Machine): ############################################################################### - """ - The pre-existing baselines root dir is used for integration testing only. - It can be an empty string (""). If so, test-all-scream will use a default - (ctest-build/baselines), and build baselines on the fly. - This directory is interpreted as the directory where different builds - subdirs are located (full_debug, full_sp_debug, debug_no_fpe). - """ - assert_machine_supported(machine) - return MACHINE_METADATA[machine][3] + concrete = True + @classmethod + def setup(cls): + super().setup_base("chrysalis") + + cls.env_setup = [f"eval $({CIMEROOT}/CIME/Tools/get_case_env)", "export OMP_NUM_THREADS=1"] + cls.batch = "srun --mpi=pmi2 -l -N 1 --kill-on-bad-exit --cpu_bind=cores" + cls.baselines_dir = "/lcrc/group/e3sm/baselines/chrys/intel/scream" + + cls.cxx_compiler = "mpic++" + cls.c_compiler = "mpicc" + cls.ftn_compiler = "mpif90" ############################################################################### -def logical_cores_per_physical_core(): +class Mappy(Machine): ############################################################################### - return psutil.cpu_count() // psutil.cpu_count(logical=False) + concrete = True + @classmethod + def setup(cls): + super().setup_base("mappy") + + cls.env_setup = ["module purge", + "module load sems-cmake/3.27.9 sems-git/2.42.0 sems-gcc/11.4.0 sems-openmpi-no-cuda/4.1.6 sems-netcdf-c/4.9.2 sems-netcdf-cxx/4.2 sems-netcdf-fortran/4.6.1 sems-parallel-netcdf/1.12.3 sems-openblas", + "export GATOR_INITIAL_MB=4000MB", + ] + cls.baselines_dir = "/sems-data-store/ACME/baselines/scream/master-baselines" ############################################################################### -def get_available_cpu_count(logical=True): +class Weaver(Machine): ############################################################################### - """ - Get number of CPUs available to this process and its children. logical=True - will include hyperthreads, logical=False will return only physical cores - """ - affinity_len = len(psutil.Process().cpu_affinity()) - if not logical: - hyperthread_ratio = logical_cores_per_physical_core() - return int(affinity_len / hyperthread_ratio) - else: - return affinity_len + concrete = True + @classmethod + def setup(cls): + super().setup_base("weaver") + + cls.env_setup = ["source /etc/profile.d/modules.sh", + "module purge", + "module load cmake/3.25.1 git/2.39.1 python/3.10.8 py-netcdf4/1.5.8 gcc/11.3.0 cuda/11.8.0 openmpi netcdf-c netcdf-fortran parallel-netcdf netlib-lapack", + "export HDF5_USE_FILE_LOCKING=FALSE" + ] + cls.baselines_dir = "/home/projects/e3sm/scream/pr-autotester/master-baselines/weaver/" + cls.batch = "bsub -I -q rhel8 -n 4 -gpu num=4" + + cls.num_run_res = 4 # four gpus + cls.gpu_arch = "cuda" + +############################################################################### +class Compy(Machine): +############################################################################### + concrete = True + @classmethod + def setup(cls): + super().setup_base("compy") + + cls.env_setup = ["module purge", "module load cmake/3.19.6 gcc/8.1.0 mvapich2/2.3.1 python/3.7.3"] + cls.batch = "srun --time 02:00:00 --nodes=1 -p short --exclusive --account e3sm" ############################################################################### -def get_mach_compilation_resources(): +class GHCISNLCPU(Machine): ############################################################################### - return get_available_cpu_count() + concrete = True + @classmethod + def setup(cls): + super().setup_base("ghci-snl-cpu") + cls.baselines_dir = "/projects/e3sm/baselines/scream/ghci-snl-cpu" + cls.env_setup = ["export GATOR_INITIAL_MB=4000MB"] ############################################################################### -def get_mach_testing_resources(machine): +class GHCISNLCuda(Machine): ############################################################################### - """ - The number of host cores is used to parallelize compilation, - while the number of devices is used to parallelize testing. - On CPU machines, the two will usually coincide, while on GPU - machines they are going to be different (compile on CPU, run on GPU). - One difference is that, for CPU machines, we allow hyperthreading for - compilation but not for testing because we want to minimize fragmentation - of jobs across cores. - """ - if is_cuda_machine(machine): - prefix = "srun " if is_salloc(machine) else "" - return int(run_cmd_no_fail(f"{prefix}nvidia-smi -L | wc -l")) - else: - return get_available_cpu_count() + concrete = True + @classmethod + def setup(cls): + super().setup_base(name="ghci-snl-cuda") + cls.baselines_dir = "/projects/e3sm/baselines/scream/ghci-snl-cuda" + cls.gpu_arch = "cuda" + cls.num_run_res = int(run_cmd_no_fail("nvidia-smi --query-gpu=name --format=csv,noheader | wc -l")) ############################################################################### -def is_salloc(machine): +class GHCIOCI(Machine): ############################################################################### - """ - Return true if we are running on an salloc'd job. - """ - bcmd = get_mach_batch_command(machine) - return "salloc" in bcmd and "srun" not in bcmd + concrete = True + @classmethod + def setup(cls): + super().setup_base(name="ghci-oci") + cls.env_setup = [f"eval $({CIMEROOT}/CIME/Tools/get_case_env -c SMS.ne4pg2_ne4pg2.F2010-SCREAMv1.ghci-oci_gnu)"] ############################################################################### -def is_cuda_machine(machine): +class Lassen(Machine): ############################################################################### - assert_machine_supported(machine) + concrete = True + @classmethod + def setup(cls): + super().setup_base("lassen") + cls.baselines_dir = "/projects/e3sm/baselines/scream/master-baselines" - env_setup_raw = MACHINE_METADATA[machine][0] - env_setup_str = " ".join(env_setup_raw).lower() + cls.env_setup = ["module --force purge", + "module load git gcc/8.3.1 cuda/11.8.0 cmake/3.16.8 spectrum-mpi python/3.7.2", + "export LLNL_USE_OMPI_VARS='y'", + "export PATH=/usr/workspace/e3sm/netcdf/bin:$PATH", + "export LD_LIBRARY_PATH=/usr/workspace/e3sm/netcdf/lib:$LD_LIBRARY_PATH", + ] + cls.batch = "bsub -Ip -qpdebug" - return ("no-cuda" not in env_setup_str and "cuda" in env_setup_str) + cls.num_run_res = 4 # four gpus + cls.gpu_arch = "cuda" + +############################################################################### +class LLNLIntel(Machine): +############################################################################### + @classmethod + def setup_llnl_intel(cls,name): + super().setup_base(name) + + cls.env_setup = ["module --force purge", + "module use --append /usr/workspace/e3sm/install/quartz/modulefiles", + "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1" + ] + cls.batch = "salloc --partition=pdebug", + + +############################################################################### +class RubyIntel(LLNLIntel): +############################################################################### + concrete = True + @classmethod + def setup(cls): + super().setup_llnl_intel("ruby-intel") + +############################################################################### +class DaneIntel(LLNLIntel): +############################################################################### + concrete = True + @classmethod + def setup(cls): + super().setup_llnl_intel("dane-intel") + +############################################################################### +class QuartzIntel(LLNLIntel): +############################################################################### + concrete = True + @classmethod + def setup(cls): + super().setup_llnl_intel("quartz-intel") + + cls.env_setup = ["module --force purge", + "module use --append /usr/workspace/e3sm/install/quartz/modulefiles", + "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1" + ] + cls.batch = "salloc --partition=pdebug", + +############################################################################### +class QuartzGCC(Machine): +############################################################################### + concrete = True + @classmethod + def setup(cls): + super().setup_base("quartz-gcc") + + cls.env_setup = ["module --force purge", + "module load StdEnv cmake/3.16.8 mkl/2019.0 gcc-8.3.1 netcdf-fortran/4.4.4 netcdf/4.4.1.1 pnetcdf/1.9.0 mvapich2/2.3" + ] + cls.batch = "salloc --partition=pdebug", + +############################################################################### +class Syrah(Machine): +############################################################################### + concrete = True + @classmethod + def setup(cls): + super().setup_base("syrah") + + cls.env_setup = ["module --force purge", + "module load StdEnv cmake/3.16.8 mkl/2019.0 intel/19.0.4 netcdf-fortran/4.4.4 netcdf/4.4.1.1 pnetcdf/1.9.0 mvapich2/2.3" + ] + cls.batch = "salloc --partition=pdebug --time=60", + +############################################################################### +class AnlGce(Machine): +############################################################################### + concrete = True + @classmethod + def setup(cls): + super().setup_base("anlgce") + + cls.env_setup = [". /nfs/gce/software/spack/opt/spack/linux-ubuntu20.04-x86_64/gcc-9.3.0/lmod-8.3-6fjdtku/lmod/lmod/init/sh", + "module purge", + "module load autoconf/2.69-bmnwajj automake/1.16.3-r7w24o4 libtool/2.4.6-uh3mpsu m4/1.4.19-7fztfyz cmake/3.20.5-zyz2eld gcc/11.1.0-qsjmpcg zlib/1.2.11-p7dmb5p", + "export LD_LIBRARY_PATH=/nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/mpich/4.0/gcc-11.1.0/lib:$LD_LIBRARY_PATH", + "export PATH=/nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/mpich/4.0/gcc-11.1.0/bin:/nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-11.1.0/bin:$PATH", + "export NetCDF_ROOT=/nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-11.1.0", + "export PERL5LIB=/nfs/gce/projects/climate/software/perl5/lib/perl5" + ] + +############################################################################### +class AnlGceUb22(Machine): +############################################################################### + concrete = True + @classmethod + def setup(cls): + super().setup_base("anlgce-ub22") + + cls.env_setup = [". /nfs/gce/software/custom/linux-ubuntu22.04-x86_64/spack/opt/spack/linux-ubuntu22.04-x86_64/gcc-11.2.0/lmod-8.5.6-hkjjxhp/lmod/lmod/init/sh", + "module purge", + "module load gcc/12.1.0", + "export LD_LIBRARY_PATH=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/mpich/4.1.2/gcc-12.1.0/lib:$LD_LIBRARY_PATH", + "export PATH=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/mpich/4.1.2/gcc-12.1.0/bin:/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-12.1.0/bin:$PATH", + "export NetCDF_ROOT=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-12.1.0", + "export PERL5LIB=/nfs/gce/projects/climate/software/perl5/lib/perl5" + ] + +############################################################################### +def get_all_machines (base=Machine): +############################################################################### + # If the user has the file ~/.cime/scream_mach_specs.py, import the machine type Local from it + if pathlib.Path("~/.cime/scream_mach_specs.py").expanduser().is_file(): # pylint: disable=no-member + sys.path.append(str(pathlib.Path("~/.cime").expanduser())) + # Add the directory of this file to sys.path, so that ~/.cime/scream_mach_specs.py, + # if present, can simply do "from machine_specs import Machine" to define machine Local + sys.path.append(os.path.dirname(__file__)) + + # Pylint does not see that this import adds one subclass to Machine + # Also, its static analysis runs on a static sys.path, without ~/.cime, + # so it doesn't find the module + from scream_mach_specs import Local# pylint: disable=unused-import, import-error + + machines = [] + for m in base.__subclasses__(): + if m.concrete: + m.setup() # Init the class static data + machines.append(m) + + machines.extend(get_all_machines(m)) + + return machines + +############################################################################### +def get_machine (name): +############################################################################### + + all_machines = get_all_machines() + + for m in all_machines: + if m.name==name: + return m + + raise ValueError(f"Machine with name '{name}' not found.") + +############################################################################### +def get_all_supported_machines (): +############################################################################### + return [m.name for m in get_all_machines()] + +############################################################################### +def is_machine_supported(machine): +############################################################################### + return machine in get_all_supported_machines() + +############################################################################### +def assert_machine_supported(machine): +############################################################################### + expect(is_machine_supported(machine), + "Machine {} is not currently supported by scream testing system.\n" + f" Currently supported machines: {','.join(get_all_supported_machines())}\n" + " Note: you can also create a file `~/.cime/scream_mach_specs.py` with your local machine specs.".format(machine)) + +############################################################################### +def get_mach_env_setup_command(machine, ctest_j=None): +############################################################################### + """ + ctest_j=None -> probe for hardware for testing resources + ctest_j=-1 -> Skip CTEST_PARALLEL_LEVEL + """ + + mach = get_machine(machine) + mach_custom_env = mach.env_setup + if ctest_j != -1: + ctest_j = mach.num_run_res if ctest_j is None else ctest_j + mach_custom_env.append("export CTEST_PARALLEL_LEVEL={}".format(ctest_j)) + + if not mach.uses_gpu(): + mach_custom_env.append("export OMP_PROC_BIND=spread") + + return mach_custom_env ############################################################################### def setup_mach_env(machine, ctest_j=None): diff --git a/components/eamxx/scripts/nc-file-init b/components/eamxx/scripts/nc-file-init index 0af21e5f1a10..7657b34b4b9a 100755 --- a/components/eamxx/scripts/nc-file-init +++ b/components/eamxx/scripts/nc-file-init @@ -5,7 +5,7 @@ Create a netcdf file for a cubed sphere geometry, with given vertical and horizontal resolution, adding requested variable (to be set to 0). """ -from utils import check_minimum_python_version +from utils import check_minimum_python_version, GoodFormatter check_minimum_python_version(3, 4) import argparse, sys, pathlib @@ -32,7 +32,7 @@ OR """.format(pathlib.Path(args[0]).name), description=description, - formatter_class=argparse.ArgumentDefaultsHelpFormatter + formatter_class=GoodFormatter ) # The file name diff --git a/components/eamxx/scripts/perf-analysis b/components/eamxx/scripts/perf-analysis index 4e71b41f97c9..f9a66c984cb4 100755 --- a/components/eamxx/scripts/perf-analysis +++ b/components/eamxx/scripts/perf-analysis @@ -5,7 +5,7 @@ Run different versions of an app and compare performance. It is expected that this will be run from micro-apps directory. """ -from utils import expect, check_minimum_python_version +from utils import expect, check_minimum_python_version, GoodFormatter check_minimum_python_version(3, 4) import argparse, sys, os, socket @@ -38,7 +38,7 @@ OR > {0} ni:64 --test=ref:"src/physics/shoc/tests/shoc_run_and_cmp_cxx -r 10 -s 30 -i NI foo" -s ni:2:8192 --cd """.format(os.path.basename(args[0])), description=description, - formatter_class=argparse.ArgumentDefaultsHelpFormatter + formatter_class=GoodFormatter ) parser.add_argument("argmap", nargs="+", help="Argument map, NAME:STARTING_VAL. First arg is assumed to be the arg/s used in the core metric") diff --git a/components/eamxx/scripts/plot b/components/eamxx/scripts/plot index 0df75b312dbd..59f962f59b17 100755 --- a/components/eamxx/scripts/plot +++ b/components/eamxx/scripts/plot @@ -4,7 +4,7 @@ Plot results from perf_analysis """ -from utils import expect, check_minimum_python_version +from utils import expect, check_minimum_python_version, GoodFormatter check_minimum_python_version(3, 5) import matplotlib.pyplot as plt @@ -34,7 +34,7 @@ OR > {0} /weaver /weaver -v commit:version """.format(os.path.basename(args[0])), description=description, - formatter_class=argparse.ArgumentDefaultsHelpFormatter + formatter_class=GoodFormatter ) parser.add_argument("datafiles", nargs="+", help="Data file to plot") diff --git a/components/eamxx/scripts/populate-nc-file b/components/eamxx/scripts/populate-nc-file index 0cc8de3d1a04..6fd9323893f2 100755 --- a/components/eamxx/scripts/populate-nc-file +++ b/components/eamxx/scripts/populate-nc-file @@ -5,7 +5,7 @@ Populates a netcdf file adding requested variables, either importing them from another file, or by computing them as function of other existing ones. """ -from utils import check_minimum_python_version +from utils import check_minimum_python_version, GoodFormatter check_minimum_python_version(3, 4) import argparse, sys, pathlib @@ -44,14 +44,14 @@ OR > ./{0} -f my_file.nc -ifile f2.nc -ivars rho T -cvars p=rho*T^1.5 -rvars rho T - \033[1;32m# Appends to existing netcdf file, regridding T from file f2.nc, using map file map.nc + \033[1;32m# Appends to existing netcdf file, regridding T from file f2.nc, using map file map.nc and deleting rho,T from the output file. Uses nco's ncremap. > ./{0} -f my_file.nc -ifile f2.nc -ivars T -mfile map.nc """.format(pathlib.Path(args[0]).name), description=description, - formatter_class=argparse.ArgumentDefaultsHelpFormatter + formatter_class=GoodFormatter ) # The name of the nc file to populate, plus auxiliary files diff --git a/components/eamxx/scripts/CMakeLists.txt b/components/eamxx/scripts/query-cf-database/CMakeLists.txt similarity index 67% rename from components/eamxx/scripts/CMakeLists.txt rename to components/eamxx/scripts/query-cf-database/CMakeLists.txt index 377452229b9d..912007606024 100644 --- a/components/eamxx/scripts/CMakeLists.txt +++ b/components/eamxx/scripts/query-cf-database/CMakeLists.txt @@ -1,8 +1,8 @@ -# Generate the source file for the CF validator and build it. +# Build the CF validator tool -set (CF_STANDARD_NAME_FILE "${PROJECT_SOURCE_DIR}/data/cf-standard-name-table.yaml" +set (CF_STANDARD_NAME_FILE "${CMAKE_CURRENT_SOURCE_DIR}/cf-standard-name-table.yaml" CACHE STRING "Location of the cf standard name yaml table") -set (CF_SCREAM_NAME_FILE "${PROJECT_SOURCE_DIR}/data/cf-scream-name-table.yaml" +set (CF_SCREAM_NAME_FILE "${CMAKE_CURRENT_SOURCE_DIR}/cf-scream-name-table.yaml" CACHE STRING "Location of the scream-specific cf name yaml table") add_executable(query-cf-database query-cf-database.cpp) @@ -10,5 +10,6 @@ target_compile_definitions(query-cf-database PUBLIC CF_STANDARD_NAME_FILE=${CF_STANDARD_NAME_FILE} CF_SCREAM_NAME_FILE=${CF_SCREAM_NAME_FILE}) +find_package (ekat HINTS ${EKAT_ROOT}) find_package (yaml-cpp HINTS ${YAML_CPP_ROOT}) target_link_libraries(query-cf-database ekat yaml-cpp) diff --git a/components/eamxx/data/cf-scream-name-table.yaml b/components/eamxx/scripts/query-cf-database/cf-scream-name-table.yaml similarity index 100% rename from components/eamxx/data/cf-scream-name-table.yaml rename to components/eamxx/scripts/query-cf-database/cf-scream-name-table.yaml diff --git a/components/eamxx/data/cf-standard-name-table.yaml b/components/eamxx/scripts/query-cf-database/cf-standard-name-table.yaml similarity index 100% rename from components/eamxx/data/cf-standard-name-table.yaml rename to components/eamxx/scripts/query-cf-database/cf-standard-name-table.yaml diff --git a/components/eamxx/scripts/cf-xml-to-yaml b/components/eamxx/scripts/query-cf-database/cf-xml-to-yaml similarity index 100% rename from components/eamxx/scripts/cf-xml-to-yaml rename to components/eamxx/scripts/query-cf-database/cf-xml-to-yaml diff --git a/components/eamxx/scripts/query-cf-database.cpp b/components/eamxx/scripts/query-cf-database/query-cf-database.cpp similarity index 100% rename from components/eamxx/scripts/query-cf-database.cpp rename to components/eamxx/scripts/query-cf-database/query-cf-database.cpp diff --git a/components/eamxx/scripts/query-cime b/components/eamxx/scripts/query-cime index 68472ac25557..46d42668823c 100755 --- a/components/eamxx/scripts/query-cime +++ b/components/eamxx/scripts/query-cime @@ -4,7 +4,7 @@ Query the CIME config_machines.xml file """ -from utils import check_minimum_python_version +from utils import check_minimum_python_version, GoodFormatter check_minimum_python_version(3, 4) import argparse, sys, pathlib @@ -24,7 +24,7 @@ OR > {0} mappy DIN_LOC_ROOT """.format(pathlib.Path(args[0]).name), description=description, - formatter_class=argparse.ArgumentDefaultsHelpFormatter + formatter_class=GoodFormatter ) parser.add_argument("machine", help="The name of the machine") diff --git a/components/eamxx/scripts/query-scream b/components/eamxx/scripts/query-scream index 240552b3b75e..b1cec87c90a1 100755 --- a/components/eamxx/scripts/query-scream +++ b/components/eamxx/scripts/query-scream @@ -4,7 +4,7 @@ Query the SCREAM machines_specs.py file """ -from utils import check_minimum_python_version +from utils import check_minimum_python_version, GoodFormatter check_minimum_python_version(3, 4) import argparse, sys, pathlib @@ -24,7 +24,7 @@ OR > {0} mappy cxx_compiler """.format(pathlib.Path(args[0]).name), description=description, - formatter_class=argparse.ArgumentDefaultsHelpFormatter + formatter_class=GoodFormatter ) parser.add_argument("machine", help="The name of the machine") diff --git a/components/eamxx/scripts/query_scream.py b/components/eamxx/scripts/query_scream.py index 4b26451c7cb2..a6fe5096bfb5 100644 --- a/components/eamxx/scripts/query_scream.py +++ b/components/eamxx/scripts/query_scream.py @@ -1,9 +1,5 @@ -from machines_specs import assert_machine_supported, \ - get_mach_cxx_compiler, get_mach_c_compiler, get_mach_f90_compiler, \ - get_mach_batch_command, get_mach_env_setup_command, \ - get_mach_baseline_root_dir, is_cuda_machine, \ - get_mach_compilation_resources, get_mach_testing_resources +from machines_specs import assert_machine_supported, get_machine, get_mach_env_setup_command from utils import expect CHOICES = ( @@ -24,23 +20,24 @@ def query_scream(machine, param): assert_machine_supported(machine) expect(param in CHOICES, f"Unknown param {param}") + mach = get_machine(machine) if param == "cxx_compiler": - return get_mach_cxx_compiler(machine) + return mach.cxx_compiler elif param == "c_compiler": - return get_mach_c_compiler(machine) + return mach.c_compiler elif param == "f90_compiler": - return get_mach_f90_compiler(machine) + return mach.ftn_compiler elif param == "batch": - return get_mach_batch_command(machine) + return mach.batch elif param == "env": return get_mach_env_setup_command(machine) elif param == "baseline_root": - return get_mach_baseline_root_dir(machine) + return mach.baselines_dir elif param == "cuda": - return str(is_cuda_machine(machine)) + return str(mach.gpu_arch == "cuda") elif param == "comp_j": - return get_mach_compilation_resources() + return num_bld_res elif param == "test_j": - return get_mach_testing_resources(machine) + return gnum_run_res else: expect(False, f"Unhandled param {param}") diff --git a/components/eamxx/scripts/scream-env-cmd b/components/eamxx/scripts/scream-env-cmd index a808c7fd5f71..34b5293807f7 100755 --- a/components/eamxx/scripts/scream-env-cmd +++ b/components/eamxx/scripts/scream-env-cmd @@ -6,7 +6,7 @@ This can be used with a bash function or the source_to_load_scream_env.sh script to conveniently load the SCREAM-approved env into your current shell. """ -from utils import check_minimum_python_version +from utils import check_minimum_python_version, GoodFormatter check_minimum_python_version(3, 4) from machines_specs import get_mach_env_setup_command @@ -41,7 +41,7 @@ OR """.format(pathlib.Path(args[0]).name), description=description, - formatter_class=argparse.ArgumentDefaultsHelpFormatter + formatter_class=GoodFormatter ) parser.add_argument("mach", help="The machine name for which you want the scream env") diff --git a/components/eamxx/scripts/scripts-ctest-driver b/components/eamxx/scripts/scripts-ctest-driver index 94250eea0de1..ba5632402c90 100755 --- a/components/eamxx/scripts/scripts-ctest-driver +++ b/components/eamxx/scripts/scripts-ctest-driver @@ -5,7 +5,7 @@ Drive ctest testing of scream's scripts. Because this includes testing of jenkins scripts, it should always be run from a login node. """ -from utils import check_minimum_python_version +from utils import check_minimum_python_version, GoodFormatter check_minimum_python_version(3, 4) import argparse, sys, pathlib @@ -26,7 +26,7 @@ OR > ./scripts/{0} -m mappy """.format(pathlib.Path(args[0]).name), description=description, - formatter_class=argparse.ArgumentDefaultsHelpFormatter + formatter_class=GoodFormatter ) parser.add_argument("-s", "--submit", action="store_true", help="Submit results to dashboad") diff --git a/components/eamxx/scripts/scripts-tests b/components/eamxx/scripts/scripts-tests index 1fd2e48c1b40..67416b23d2b5 100755 --- a/components/eamxx/scripts/scripts-tests +++ b/components/eamxx/scripts/scripts-tests @@ -10,11 +10,11 @@ If you are on a batch machine, it is expected that you are on a compute node. TODO: Add doctests to libs """ -from utils import check_minimum_python_version, expect, ensure_pylint, run_cmd_no_fail, run_cmd_assert_result +from utils import check_minimum_python_version, expect, ensure_pylint, run_cmd_no_fail, run_cmd_assert_result, GoodFormatter check_minimum_python_version(3, 6) -from machines_specs import is_machine_supported, is_cuda_machine +from machines_specs import is_machine_supported, get_machine from git_utils import get_current_branch, get_current_commit, get_current_head, git_refs_difference, \ is_repo_clean, get_common_ancestor, checkout_git_ref, get_git_toplevel_dir @@ -34,12 +34,12 @@ CONFIG = { def test_cmake_cache_contents(test_obj, build_name, cache_var, expected_value): ############################################################################### cache_file = TEST_DIR.parent.joinpath("ctest-build", build_name, "CMakeCache.txt") - test_obj.assertTrue(cache_file.is_file(), "Missing cache file {}".format(cache_file)) # pylint: disable=no-member + test_obj.assertTrue(cache_file.is_file(), f"Missing cache file {cache_file}") # pylint: disable=no-member - grep_output = run_cmd_assert_result(test_obj, "grep ^{} CMakeCache.txt".format(cache_var), from_dir=cache_file.parent) + grep_output = run_cmd_assert_result(test_obj, f"grep ^{cache_var} CMakeCache.txt", from_dir=cache_file.parent) value = grep_output.split("=", maxsplit=1)[-1] test_obj.assertEqual(expected_value.upper(), value.upper(), - msg="For CMake cache variable {}, expected value '{}', got '{}'".format(cache_var, expected_value, value)) + msg=f"For CMake cache variable {cache_var}, expected value '{expected_value}', got '{value}'") ############################################################################### def test_baseline_has_sha(test_obj, test_dir, build_name, expected_sha): @@ -58,8 +58,8 @@ def add_file_and_commit(repo, filename, contents, commit_msg): with filepath.open("w", encoding="utf-8") as fd: fd.write(contents) - run_cmd_no_fail("git add {}".format(filename), from_dir=repo) - run_cmd_no_fail("git commit -m {}".format(commit_msg), from_dir=repo) + run_cmd_no_fail(f"git add {filename}", from_dir=repo) + run_cmd_no_fail(f"git commit -m {commit_msg}", from_dir=repo) ############################################################################### def test_branch_switch(test_obj, repo, branch_name): @@ -116,7 +116,7 @@ class TestBaseOuter: # Hides the TestBase class from test scanner super(TestBaseOuter.TestBase, self).__init__(*internal_args) self._source_file = source_file self._cmds = list(cmds) - self._machine = CONFIG["machine"] + self._machine = get_machine(CONFIG["machine"]) self._full = CONFIG["full"] self._jenkins = CONFIG["jenkins"] @@ -126,14 +126,14 @@ class TestBaseOuter: # Hides the TestBase class from test scanner self._results.mkdir(parents=True, exist_ok=True) # pylint: disable=no-member def get_cmd(self, cmd, machine): - return cmd.replace("$machine", machine).replace("$results", str(self._results)) + return cmd.replace("$machine", machine.name).replace("$results", str(self._results)) def test_doctests(self): - run_cmd_assert_result(self, "python3 -m doctest {}".format(self._source_file), from_dir=TEST_DIR) + run_cmd_assert_result(self, f"python3 -m doctest {self._source_file}", from_dir=TEST_DIR) def test_pylint(self): ensure_pylint() - run_cmd_assert_result(self, "python3 -m pylint --disable C --disable R {}".format(self._source_file), from_dir=TEST_DIR) + run_cmd_assert_result(self, f"python3 -m pylint --disable C --disable R {self._source_file}", from_dir=TEST_DIR) def test_full(self): if self._full: @@ -231,13 +231,13 @@ class TestTestAllScream(TestBaseOuter.TestBase): """ if not self._jenkins: options = "-t dbg --config-only" - cmd = self.get_cmd("./test-all-scream -m $machine {}".format(options), + cmd = self.get_cmd(f"./test-all-scream -m $machine {options}", self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_cmake_cache_contents(self, "full_debug", "CMAKE_BUILD_TYPE", "Debug") test_cmake_cache_contents(self, "full_debug", "SCREAM_DOUBLE_PRECISION", "TRUE") test_cmake_cache_contents(self, "full_debug", "SCREAM_FPE", "FALSE") - if is_cuda_machine(self._machine): + if self._machine.uses_gpu(): test_cmake_cache_contents(self, "full_debug", "SCREAM_PACK_SIZE", "1") else: test_cmake_cache_contents(self, "full_debug", "SCREAM_PACK_SIZE", "16") @@ -250,13 +250,13 @@ class TestTestAllScream(TestBaseOuter.TestBase): """ if not self._jenkins: options = "-t sp --config-only" - cmd = self.get_cmd("./test-all-scream -m $machine {}".format(options), + cmd = self.get_cmd(f"./test-all-scream -m $machine {options}", self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_cmake_cache_contents(self, "full_sp_debug", "CMAKE_BUILD_TYPE", "Debug") test_cmake_cache_contents(self, "full_sp_debug", "SCREAM_DOUBLE_PRECISION", "FALSE") test_cmake_cache_contents(self, "full_sp_debug", "SCREAM_FPE", "FALSE") - if is_cuda_machine(self._machine): + if self._machine.uses_gpu(): test_cmake_cache_contents(self, "full_sp_debug", "SCREAM_PACK_SIZE", "1") else: test_cmake_cache_contents(self, "full_sp_debug", "SCREAM_PACK_SIZE", "16") @@ -268,11 +268,11 @@ class TestTestAllScream(TestBaseOuter.TestBase): Test the 'fpe' test in test-all-scream. It should set certain CMake values """ if not self._jenkins: - if is_cuda_machine(self._machine): - self.skipTest("Skipping FPE check on cuda") + if self._machine.uses_gpu(): + self.skipTest("Skipping FPE check on gpu machines") else: options = "-t fpe --config-only" - cmd = self.get_cmd("./test-all-scream -m $machine {}".format(options), + cmd = self.get_cmd(f"./test-all-scream -m $machine {options}", self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_cmake_cache_contents(self, "debug_nopack_fpe", "CMAKE_BUILD_TYPE", "Debug") @@ -288,16 +288,17 @@ class TestTestAllScream(TestBaseOuter.TestBase): """ if not self._jenkins: options = "-t mem --config-only" - cmd = self.get_cmd("./test-all-scream -m $machine {}".format(options), + cmd = self.get_cmd(f"./test-all-scream -m $machine {options}", self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) - builddir = "compute_sanitizer_memcheck" if is_cuda_machine(self._machine) else "valgrind" - test_cmake_cache_contents(self, builddir, "CMAKE_BUILD_TYPE", "Debug") + builddir = "compute_sanitizer_memcheck" if self._machine.gpu_arch=="cuda" else "valgrind" test_cmake_cache_contents(self, builddir, "SCREAM_TEST_SIZE", "SHORT") - if is_cuda_machine(self._machine): + if self._machine.gpu_arch=="cuda": + test_cmake_cache_contents(self, builddir, "CMAKE_BUILD_TYPE", "Debug") test_cmake_cache_contents(self, builddir, "EKAT_ENABLE_COMPUTE_SANITIZER", "TRUE") test_cmake_cache_contents(self, builddir, "EKAT_COMPUTE_SANITIZER_OPTIONS", "--tool=memcheck") else: + test_cmake_cache_contents(self, builddir, "CMAKE_BUILD_TYPE", "RelWithDebInfo") test_cmake_cache_contents(self, builddir, "EKAT_ENABLE_VALGRIND", "TRUE") else: self.skipTest("Skipping config-only run for jenkins test") @@ -308,7 +309,7 @@ class TestTestAllScream(TestBaseOuter.TestBase): """ if not self._jenkins: options = "-t opt --config-only" - cmd = self.get_cmd("./test-all-scream -m $machine {}".format(options), + cmd = self.get_cmd(f"./test-all-scream -m $machine {options}", self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_cmake_cache_contents(self, "release", "CMAKE_BUILD_TYPE", "Release") @@ -439,7 +440,7 @@ class TestTestAllScream(TestBaseOuter.TestBase): if self._jenkins: # We set PULLREQUESTNUM to block dashboard submission # We set SCREAM_FAKE_AUTO to not interere with real baselines - cmd = self.get_cmd("PR_LABELS= NODE_NAME={} SCREAM_FAKE_AUTO=TRUE PULLREQUESTNUM=42 ./jenkins/jenkins_common.sh".format(self._machine), + cmd = self.get_cmd(f"PR_LABELS= NODE_NAME={self._machine.name} SCREAM_FAKE_AUTO=TRUE PULLREQUESTNUM=42 ./jenkins/jenkins_common.sh", self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) @@ -453,7 +454,7 @@ class TestTestAllScream(TestBaseOuter.TestBase): if self._jenkins: # We set PULLREQUESTNUM to block dashboard submission # We set SCREAM_FAKE_AUTO to not interere with real baselines - cmd = self.get_cmd("PR_LABELS= NODE_NAME={} SCREAM_FAKE_AUTO=TRUE PULLREQUESTNUM= ./jenkins/jenkins_common.sh".format(self._machine), + cmd = self.get_cmd(f"PR_LABELS= NODE_NAME={self._machine.name} SCREAM_FAKE_AUTO=TRUE PULLREQUESTNUM= ./jenkins/jenkins_common.sh", self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) @@ -466,7 +467,7 @@ class TestTestAllScream(TestBaseOuter.TestBase): """ if self._jenkins: # Any fail will do, we already checked test-all-scream captures all the fail types - cmd = self.get_cmd("PR_LABELS= SCREAM_FORCE_CONFIG_FAIL=True NODE_NAME={} SCREAM_FAKE_AUTO=TRUE PULLREQUESTNUM=42 ./jenkins/jenkins_common.sh".format(self._machine), + cmd = self.get_cmd(f"PR_LABELS= SCREAM_FORCE_CONFIG_FAIL=True NODE_NAME={self._machine.name} SCREAM_FAKE_AUTO=TRUE PULLREQUESTNUM=42 ./jenkins/jenkins_common.sh", self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR, expect_works=False) @@ -479,7 +480,7 @@ class TestTestAllScream(TestBaseOuter.TestBase): """ if self._jenkins: # Any fail will do, we already checked test-all-scream captures all the fail types - cmd = self.get_cmd("PR_LABELS= SCREAM_FORCE_CONFIG_FAIL=True NODE_NAME={} SCREAM_FAKE_AUTO=TRUE PULLREQUESTNUM= ./jenkins/jenkins_common.sh".format(self._machine), + cmd = self.get_cmd(f"PR_LABELS= SCREAM_FORCE_CONFIG_FAIL=True NODE_NAME={self._machine} SCREAM_FAKE_AUTO=TRUE PULLREQUESTNUM= ./jenkins/jenkins_common.sh", self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR, expect_works=False) @@ -544,7 +545,7 @@ OR parser = argparse.ArgumentParser( usage=help_str, description=desc, - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + formatter_class=GoodFormatter) parser.add_argument("-m", "--machine", help="Provide machine name. This is required for full (not dry) runs") @@ -567,7 +568,7 @@ def scripts_tests(machine=None, full=False, jenkins=False): # Store test params in environment if machine: - expect(is_machine_supported(machine), "Machine {} is not supported".format(machine)) + expect(is_machine_supported(machine), f"Machine {machine} is not supported") CONFIG["machine"] = machine CONFIG["jenkins"] = jenkins diff --git a/components/eamxx/scripts/scripts_ctest_driver.py b/components/eamxx/scripts/scripts_ctest_driver.py index f1b320d763f7..5485dd63308f 100644 --- a/components/eamxx/scripts/scripts_ctest_driver.py +++ b/components/eamxx/scripts/scripts_ctest_driver.py @@ -1,5 +1,5 @@ from utils import run_cmd, expect, check_minimum_python_version -from machines_specs import is_machine_supported, setup_mach_env +from machines_specs import is_machine_supported, setup_mach_env, get_machine check_minimum_python_version(3, 4) diff --git a/components/eamxx/scripts/test-all-scream b/components/eamxx/scripts/test-all-scream index 3dc820581f71..da388278663a 100755 --- a/components/eamxx/scripts/test-all-scream +++ b/components/eamxx/scripts/test-all-scream @@ -15,7 +15,9 @@ Baselines: By default, test-all-scream will not run baseline tests. If you set the location specified by the machine spec. You can regenerate baselines any time by using the -g flag, but be aware this will impact everyone if you regenerate the public baselines. You can change the target baseline area -using -b $path. If -g is provided, no tests will be run; -g means generate only. +using -b $path. You can also use the magic word "LOCAL" to have test-all-scream +pick a local directory for you if you want to manage your own baselines within +the current repo. If -g is provided, no tests will be run; -g means generate only. The general workflow for baseline-changing PRs is: 1) Issue PR @@ -32,7 +34,7 @@ do for create_test tests, the only difference is that test-all-scream does basel comparison tests by default. """ -from utils import check_minimum_python_version +from utils import check_minimum_python_version, GoodFormatter check_minimum_python_version(3, 4) import argparse, sys, pathlib @@ -70,7 +72,7 @@ OR > ./scripts/{0} -m mappy -t dbg """.format(pathlib.Path(args[0]).name), description=description, - formatter_class=argparse.ArgumentDefaultsHelpFormatter + formatter_class=GoodFormatter ) parser.add_argument("-cxx","--cxx-compiler", help="C++ compiler", default=None) diff --git a/components/eamxx/scripts/test_all_scream.py b/components/eamxx/scripts/test_all_scream.py index 61ce833e5640..81dd2f4cb7c2 100644 --- a/components/eamxx/scripts/test_all_scream.py +++ b/components/eamxx/scripts/test_all_scream.py @@ -1,13 +1,11 @@ from utils import run_cmd, run_cmd_no_fail, expect, check_minimum_python_version, ensure_psutil, \ - SharedArea, safe_copy + SharedArea from git_utils import get_current_head, get_current_commit from test_factory import create_tests, COV, CSR -from machines_specs import get_mach_compilation_resources, get_mach_testing_resources, \ - get_mach_baseline_root_dir, setup_mach_env, is_cuda_machine, \ - get_mach_cxx_compiler, get_mach_f90_compiler, get_mach_c_compiler, is_machine_supported, \ - logical_cores_per_physical_core +from machines_specs import get_machine, setup_mach_env, is_machine_supported, \ + logical_cores_per_physical_core, get_cpu_ids_from_slurm_env_var check_minimum_python_version(3, 4) @@ -49,8 +47,6 @@ def __init__(self, cxx_compiler=None, f90_compiler=None, c_compiler=None, self._c_compiler = c_compiler self._submit = submit self._parallel = parallel - self._machine = machine - self._local = local self._config_only = config_only self._baseline_dir = baseline_dir self._custom_cmake_opts = custom_cmake_opts @@ -78,18 +74,19 @@ def __init__(self, cxx_compiler=None, f90_compiler=None, c_compiler=None, "Makes no sense to ask for --quick-rerun and --config-only at the same time") # Probe machine if none was specified - if self._machine is None: + if machine is not None: + self._machine = get_machine(machine) + expect (not local, "Specifying a machine while passing '-l,--local' is ambiguous.") + else: # We could potentially integrate more with CIME here to do actual # nodename probing. if "SCREAM_MACHINE" in os.environ and is_machine_supported(os.environ["SCREAM_MACHINE"]): - self._machine = os.environ["SCREAM_MACHINE"] + self._machine = get_machine(os.environ["SCREAM_MACHINE"]) else: - expect(self._local, + expect(local, "test-all-scream requires either the machine arg (-m $machine) or the -l flag," "which makes it look for machine specs in '~/.cime/scream_mach_specs.py'.") - self._machine = "local" - else: - expect (not self._local, "Specifying a machine while passing '-l,--local' is ambiguous.") + self._machine = get_machine("local") # Compute root dir (where repo is) and work dir (where build/test will happen) if not self._root_dir: @@ -101,7 +98,7 @@ def __init__(self, cxx_compiler=None, f90_compiler=None, c_compiler=None, # Make our test objects! Change mem to default mem-check test for current platform if "mem" in tests: - tests[tests.index("mem")] = "csm" if self.on_cuda() else "valg" + tests[tests.index("mem")] = "csm" if self._machine.gpu_arch=="cuda" else "valg" self._tests = create_tests(tests, self) if self._work_dir is not None: @@ -118,7 +115,7 @@ def __init__(self, cxx_compiler=None, f90_compiler=None, c_compiler=None, ################################### # Deduce how many compilation resources per test - make_max_jobs = get_mach_compilation_resources() + make_max_jobs = self._machine.num_bld_res if make_parallel_level > 0: expect(make_parallel_level <= make_max_jobs, f"Requested make_parallel_level {make_parallel_level} is more than max available {make_max_jobs}") @@ -127,7 +124,7 @@ def __init__(self, cxx_compiler=None, f90_compiler=None, c_compiler=None, else: print(f"Note: no value passed for --make-parallel-level. Using the default for this machine: {make_max_jobs}") - ctest_max_jobs = get_mach_testing_resources(self._machine) + ctest_max_jobs = self._machine.num_run_res if ctest_parallel_level > 0: expect(ctest_parallel_level <= ctest_max_jobs, f"Requested ctest_parallel_level {ctest_parallel_level} is more than max available {ctest_max_jobs}") @@ -155,7 +152,7 @@ def __init__(self, cxx_compiler=None, f90_compiler=None, c_compiler=None, # Avoid splitting physical cores across test types make_jobs_per_test = ((make_max_jobs // len(self._tests)) // log_per_phys) * log_per_phys - if self.on_cuda(): + if self._machine.uses_gpu(): ctest_jobs_per_test = ctest_max_jobs // len(self._tests) else: ctest_jobs_per_test = ((ctest_max_jobs // len(self._tests)) // log_per_phys) * log_per_phys @@ -176,7 +173,7 @@ def __init__(self, cxx_compiler=None, f90_compiler=None, c_compiler=None, # Need to happen before compiler probing if not self._preserve_env: # Setup the env on this machine - setup_mach_env(self._machine, ctest_j=ctest_max_jobs) + setup_mach_env(self._machine.name, ctest_j=ctest_max_jobs) ############################################ # Check repo status # @@ -193,7 +190,7 @@ def __init__(self, cxx_compiler=None, f90_compiler=None, c_compiler=None, # These two dir are special dir for "on-the-fly baselines" and "machine's official baselines" local_baseline_dir = self._work_dir/"baselines" - auto_dir = Path(get_mach_baseline_root_dir(self._machine)).absolute() + auto_dir = Path(self._machine.baselines_dir).expanduser().absolute() # Handle the "fake" auto case, used in scripts tests if "SCREAM_FAKE_AUTO" in os.environ: auto_dir = auto_dir / "fake" @@ -224,11 +221,11 @@ def __init__(self, cxx_compiler=None, f90_compiler=None, c_compiler=None, ############################################ if self._cxx_compiler is None: - self._cxx_compiler = get_mach_cxx_compiler(self._machine) + self._cxx_compiler = self._machine.cxx_compiler if self._f90_compiler is None: - self._f90_compiler = get_mach_f90_compiler(self._machine) + self._f90_compiler = self._machine.ftn_compiler if self._c_compiler is None: - self._c_compiler = get_mach_c_compiler(self._machine) + self._c_compiler = self._machine.c_compiler ############################################################################### def create_tests_dirs(self, root, clean): @@ -276,11 +273,6 @@ def get_machine(self): ############################################################################### return self._machine - ########################################################################### - def on_cuda(self): - ########################################################################### - return is_cuda_machine(self._machine) - ############################################################################### def get_preexisting_baseline(self, test): ############################################################################### @@ -313,20 +305,12 @@ def check_baselines_are_present(self): return missing - ############################################################################### - def get_machine_file(self): - ############################################################################### - if self._local: - return Path("~/.cime/scream_mach_file.cmake").expanduser() - else: - return self._root_dir/"cmake"/"machine-files"/f"{self._machine}.cmake" - ############################################################################### def generate_cmake_config(self, test, for_ctest=False): ############################################################################### # Ctest only needs config options, and doesn't need the leading 'cmake ' - result = f"{'' if for_ctest else 'cmake '}-C {self.get_machine_file()}" + result = f"{'' if for_ctest else 'cmake '}-C {self._machine.mach_file}" # Netcdf should be available. But if the user is doing a testing session # where all netcdf-related code is disabled, he/she should be able to run @@ -402,9 +386,11 @@ def get_taskset_resources(self, test, for_compile=True): ############################################################################### res_name = "compile_res_count" if for_compile else "testing_res_count" - if not for_compile and self.on_cuda(): + if not for_compile and self._machine.uses_gpu(): # For GPUs, the cpu affinity is irrelevant. Just assume all GPUS are open affinity_cp = list(range(self._ctest_max_jobs)) + elif "SLURM_CPU_BIND_LIST" in os.environ: + affinity_cp = get_cpu_ids_from_slurm_env_var() else: this_process = psutil.Process() affinity_cp = list(this_process.cpu_affinity()) @@ -492,7 +478,7 @@ def generate_ctest_config(self, cmake_config, extra_configs, test): if self._limit_test_regex: result += f"-DINCLUDE_REGEX={self._limit_test_regex} " - result += f'-S {self._root_dir}/cmake/ctest_script.cmake -DCTEST_SITE={self._machine} -DCMAKE_COMMAND="{cmake_config}" ' + result += f'-S {self._root_dir}/cmake/ctest_script.cmake -DCTEST_SITE={self._machine.name} -DCMAKE_COMMAND="{cmake_config}" ' # Ctest can only competently manage test pinning across a single instance of ctest. For # multiple concurrent instances of ctest, we have to help it. It's OK to use the compile_res_count @@ -511,7 +497,7 @@ def generate_baselines(self, test): f"Something is off. generate_baseline should have not be called for test {test}") baseline_dir = self.get_test_dir(self._baseline_dir, test) - test_dir = self.get_test_dir(self._work_dir / "tas_baseline_build", test) + test_dir = self.get_test_dir(self._work_dir, test) if test_dir.exists(): shutil.rmtree(test_dir) test_dir.mkdir() @@ -565,7 +551,7 @@ def generate_baselines(self, test): if fn != "": src = Path(fn) dst = baseline_dir / "data" / src.name - safe_copy(src, dst) + shutil.copyfile(src, dst) # Store the sha used for baselines generation. This is only for record # keeping. @@ -590,11 +576,6 @@ def generate_all_baselines(self): print(f"Generating baselines using '{git_head}'") print("###############################################################################") - tas_baseline_bld = self._work_dir / "tas_baseline_build" - if tas_baseline_bld.exists(): - shutil.rmtree(tas_baseline_bld) - tas_baseline_bld.mkdir() - success = True num_workers = len(tests_needing_baselines) if self._parallel else 1 with threading3.ProcessPoolExecutor(max_workers=num_workers) as executor: diff --git a/components/eamxx/scripts/test_factory.py b/components/eamxx/scripts/test_factory.py index f85461ad23b4..47ae47367218 100644 --- a/components/eamxx/scripts/test_factory.py +++ b/components/eamxx/scripts/test_factory.py @@ -115,7 +115,7 @@ def __init__(self, tas): [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_DEFAULT_BFB", "True"), ("SCREAM_PACK_SIZE", "1"), ("SCREAM_FPE","True")], uses_baselines=False, - on_by_default=(tas is not None and not tas.on_cuda()) + on_by_default=(tas is not None and not tas._machine.uses_gpu()) ) ############################################################################### @@ -153,15 +153,18 @@ def __init__(self, tas): TestProperty.__init__( self, "valgrind", - "debug with valgrind", - [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_ENABLE_VALGRIND", "True")], + "Release build where tests run through valgrind", + [("CMAKE_BUILD_TYPE", "RelWithDebInfo"), + ("EKAT_ENABLE_VALGRIND", "True"), + ("SCREAM_PACK_SIZE", "1"), + ("SCREAM_TEST_MAX_THREADS", "2")], uses_baselines=False, on_by_default=False, default_test_len="short" ) if tas is not None: # If a stored suppression file exists for this machine, use it - persistent_supp_file = tas.get_root_dir() / "scripts" / "jenkins" / "valgrind" / f"{tas.get_machine()}.supp" + persistent_supp_file = tas.get_root_dir() / "scripts" / "jenkins" / "valgrind" / f"{tas.get_machine().name}.supp" if persistent_supp_file.exists(): self.cmake_args.append( ("EKAT_VALGRIND_SUPPRESSION_FILE", str(persistent_supp_file)) ) diff --git a/components/eamxx/scripts/utils.py b/components/eamxx/scripts/utils.py index 9aafd09ae8ae..4788f5ff9725 100644 --- a/components/eamxx/scripts/utils.py +++ b/components/eamxx/scripts/utils.py @@ -2,11 +2,9 @@ Utilities """ -import os, sys, re, signal, subprocess, site, time, shutil +import os, sys, re, signal, subprocess, site, time, argparse from importlib import import_module -import stat as statlib from pathlib import Path -from distutils import file_util # pylint: disable=deprecated-module ############################################################################### def expect(condition, error_msg, exc_type=SystemExit, error_prefix="ERROR:"): @@ -419,79 +417,12 @@ def ensure_psutil(): _ensure_pylib_impl("psutil") def ensure_netcdf4(): _ensure_pylib_impl("netCDF4") ############################################################################### -def safe_copy(src_path, tgt_path, preserve_meta=True): -############################################################################### - """ - A flexbile and safe copy routine. Will try to copy file and metadata, but this - can fail if the current user doesn't own the tgt file. A fallback data-only copy is - attempted in this case. Works even if overwriting a read-only file. - - tgt_path can be a directory, src_path must be a file - - most of the complexity here is handling the case where the tgt_path file already - exists. This problem does not exist for the tree operations so we don't need to wrap those. - - preserve_meta toggles if file meta-data, like permissions, should be preserved. If you are - copying baseline files, you should be within a SharedArea context manager and preserve_meta - should be false so that the umask set up by SharedArea can take affect regardless of the - permissions of the src files. - """ - - # Only works for str paths for now - src_path = str(src_path) - tgt_path = str(tgt_path) - - tgt_path = ( - os.path.join(tgt_path, os.path.basename(src_path)) - if os.path.isdir(tgt_path) - else tgt_path - ) - - # Handle pre-existing file - if os.path.isfile(tgt_path): - st = os.stat(tgt_path) - owner_uid = st.st_uid - - # Handle read-only files if possible - if not os.access(tgt_path, os.W_OK): - if owner_uid == os.getuid(): - # I am the owner, make writeable - os.chmod(tgt_path, st.st_mode | statlib.S_IWRITE) - else: - # I won't be able to copy this file - raise OSError( - "Cannot copy over file {}, it is readonly and you are not the owner".format( - tgt_path - ) - ) - - if owner_uid == os.getuid(): - # I am the owner, copy file contents, permissions, and metadata - file_util.copy_file( - src_path, - tgt_path, - preserve_mode=preserve_meta, - preserve_times=preserve_meta, - verbose=0, - ) - else: - # I am not the owner, just copy file contents - shutil.copyfile(src_path, tgt_path) - - else: - # We are making a new file, copy file contents, permissions, and metadata. - # This can fail if the underlying directory is not writable by current user. - file_util.copy_file( - src_path, - tgt_path, - preserve_mode=preserve_meta, - preserve_times=preserve_meta, - verbose=0, - ) - - # If src file was executable, then the tgt file should be too - st = os.stat(tgt_path) - if os.access(src_path, os.X_OK) and st.st_uid == os.getuid(): - os.chmod( - tgt_path, st.st_mode | statlib.S_IXUSR | statlib.S_IXGRP | statlib.S_IXOTH - ) +class GoodFormatter( + argparse.ArgumentDefaultsHelpFormatter, + argparse.RawDescriptionHelpFormatter +): +############################################################################### + """ + We want argument default info to be added but we also want to + preserve formatting in the description string. + """ diff --git a/components/eamxx/src/control/atmosphere_driver.cpp b/components/eamxx/src/control/atmosphere_driver.cpp index a4bd4fabc528..23da9630b273 100644 --- a/components/eamxx/src/control/atmosphere_driver.cpp +++ b/components/eamxx/src/control/atmosphere_driver.cpp @@ -4,6 +4,7 @@ #include "physics/share/physics_constants.hpp" +#include "share/scream_config.hpp" #include "share/atm_process/atmosphere_process_group.hpp" #include "share/atm_process/atmosphere_process_dag.hpp" #include "share/field/field_utils.hpp" @@ -153,37 +154,51 @@ init_scorpio(const int atm_id) } void AtmosphereDriver:: -init_time_stamps (const util::TimeStamp& run_t0, const util::TimeStamp& case_t0) +init_time_stamps (const util::TimeStamp& run_t0, const util::TimeStamp& case_t0, int run_type) { m_atm_logger->info(" [EAMxx] Run start time stamp: " + run_t0.to_string()); m_atm_logger->info(" [EAMxx] Case start time stamp: " + case_t0.to_string()); + EKAT_REQUIRE_MSG (case_t0<=run_t0, + "Error! Case t0 time stamp must precede the run t0 time stamp.\n" + " - case t0: " + case_t0.to_string() + "\n" + " - run t0: " + run_t0.to_string() + "\n"); + // Initialize time stamps m_run_t0 = m_current_ts = run_t0; m_case_t0 = case_t0; -} - + switch (run_type) { + case 0: + m_run_type = RunType::Initial; break; + case 1: + m_run_type = RunType::Restart; break; + case -1: + m_run_type = case_t0==run_t0 ? RunType::Initial : RunType::Restart; break; + default: + EKAT_ERROR_MSG ("Unsupported/unrecognized run_type: " + std::to_string(run_type) + "\n"); + } +} void AtmosphereDriver:: -setup_iop () +setup_iop_data_manager () { // At this point, must have comm, params, initialized timestamps created. check_ad_status(s_comm_set | s_params_set | s_ts_inited); // Check to make sure iop is not already initialized - EKAT_REQUIRE_MSG(not m_iop, "Error! setup_iop() is called, but IOP already set up.\n"); + EKAT_REQUIRE_MSG(not m_iop_data_manager, "Error! setup_iop_data_manager() is called, but IOP already set up.\n"); // This function should only be called if we are enabling IOP const bool enable_iop = m_atm_params.sublist("driver_options").get("enable_iop", false); - EKAT_REQUIRE_MSG(enable_iop, "Error! setup_iop() is called, but enable_iop=false " + EKAT_REQUIRE_MSG(enable_iop, "Error! setup_iop_data_manager() is called, but enable_iop=false " "in driver_options parameters.\n"); // Params must include iop_options sublist. const auto iop_sublist_exists = m_atm_params.isSublist("iop_options"); EKAT_REQUIRE_MSG(iop_sublist_exists, - "Error! setup_iop() is called, but no iop_options " + "Error! setup_iop_data_manager() is called, but no iop_options " "defined in parameters.\n"); const auto iop_params = m_atm_params.sublist("iop_options"); @@ -192,15 +207,15 @@ setup_iop () const auto hyam = phys_grid->get_geometry_data("hyam"); const auto hybm = phys_grid->get_geometry_data("hybm"); - m_iop = std::make_shared(m_atm_comm, - iop_params, - m_run_t0, - nlevs, - hyam, - hybm); + m_iop_data_manager = std::make_shared(m_atm_comm, + iop_params, + m_run_t0, + nlevs, + hyam, + hybm); // Set IOP object in atm processes - m_atm_process_group->set_iop(m_iop); + m_atm_process_group->set_iop_data_manager(m_iop_data_manager); } void AtmosphereDriver::create_atm_processes() @@ -243,10 +258,11 @@ void AtmosphereDriver::create_grids() // To avoid having to pass the same data twice in the input file, // we have the AD add the IC file name to the GM params const auto& ic_pl = m_atm_params.sublist("initial_conditions"); - if (m_case_t0 read geo data from restart file - const auto& casename = ic_pl.get("restart_casename"); - auto filename = find_filename_in_rpointer (casename,true,m_atm_comm,m_run_t0); + const auto& provenance = m_atm_params.sublist("provenance"); + const auto& casename = provenance.get("rest_caseid"); + auto filename = find_filename_in_rpointer (casename+".scream",true,m_atm_comm,m_run_t0); gm_params.set("ic_filename", filename); m_atm_params.sublist("provenance").set("initial_conditions_file",filename); } else if (ic_pl.isParameter("Filename")) { @@ -280,7 +296,7 @@ void AtmosphereDriver::create_grids() const bool enable_iop = m_atm_params.sublist("driver_options").get("enable_iop", false); if (enable_iop) { - setup_iop (); + setup_iop_data_manager (); } // Set the grids in the processes. Do this by passing the grids manager. @@ -670,63 +686,112 @@ void AtmosphereDriver::create_fields() fm->add_to_group(fid.name(),"RESTART"); } + auto& driver_options_pl = m_atm_params.sublist("driver_options"); + const int verb_lvl = driver_options_pl.get("atmosphere_dag_verbosity_level",-1); + if (verb_lvl>0) { + // now that we've got fields, generate a DAG with fields and dependencies + // NOTE: at this point, fields provided by initial conditions may (will) + // appear as unmet dependencies + AtmProcDAG dag; + // First, add all atm processes + dag.create_dag(*m_atm_process_group); + // Write a dot file for visualizing the DAG + if (m_atm_comm.am_i_root()) { + std::string filename = "scream_atm_createField_dag"; + if (is_scream_standalone()) { + filename += ".np" + std::to_string(m_atm_comm.size()); + } + filename += ".dot"; + dag.write_dag(filename, verb_lvl); + } + } + m_ad_status |= s_fields_created; + // If the user requested it, we can save a dictionary of the FM fields to file + if (driver_options_pl.get("save_field_manager_content",false)) { + auto pg = m_grids_manager->get_grid("Physics"); + const auto& fm = m_field_mgrs.at(pg->name()); + ekat::ParameterList pl_out("field_manager_content"); + pl_out.sublist("provenance") = m_atm_params.sublist("provenance"); + DefaultMetadata std_names; + std::string desc; + desc = "content of the EAMxx FieldManager corresponding to the 'Physics' grid.\n" + "The dict keys are the field names as used in EAMxx.\n" + "For each field, we add the following entries:\n" + " - standard_name: the name commonly used to refer to this field in atm sciences (if applicable)\n" + " - units: the units for this field used in EAMxx\n" + " - layout: the names of the dimensions for this field (time excluded)\n" + " - providers: the atm processes that update/compute this field\n" + " - customers: the atm processes that require this field as an input\n"; + pl_out.set("description", desc); + auto& dict = pl_out.sublist("fields"); + for (const auto& it : *fm) { + const auto& fid = it.second->get_header().get_identifier(); + auto& pl = dict.sublist(fid.name()); + + pl.set("units",fid.get_units().to_string()); + pl.set("layout",fid.get_layout().names()); + pl.set("standard_name",std_names.get_standardname(fid.name())); + std::vector providers,customers; + const auto& track = it.second->get_header().get_tracking(); + for (auto ap : track.get_providers()) { + providers.push_back(ap.lock()->name()); + } + for (auto ap : track.get_customers()) { + customers.push_back(ap.lock()->name()); + } + pl.set("providers",providers); + pl.set("customers",customers); + } + + ekat::write_yaml_file("eamxx_field_manager_content.yaml",pl_out); + } + stop_timer("EAMxx::create_fields"); stop_timer("EAMxx::init"); m_atm_logger->info("[EAMxx] create_fields ... done!"); } -void AtmosphereDriver::initialize_output_managers () { - m_atm_logger->info("[EAMxx] initialize_output_managers ..."); +void AtmosphereDriver::create_output_managers () { + m_atm_logger->info("[EAMxx] create_output_managers ..."); start_timer("EAMxx::init"); - start_timer("EAMxx::initialize_output_managers"); + start_timer("EAMxx::create_output_managers"); - check_ad_status (s_comm_set | s_params_set | s_grids_created | s_fields_created); + check_ad_status (s_comm_set | s_params_set | s_ts_inited); auto& io_params = m_atm_params.sublist("Scorpio"); - // IMPORTANT: create model restart OutputManager first! This OM will be in charge - // of creating rpointer.atm, while other OM's will simply append to it. - // If this assumption is not verified, we must always append to rpointer, which - // can make the rpointer file a bit confusing. - - // Check for model restart output ekat::ParameterList checkpoint_params; checkpoint_params.set("frequency_units",std::string("never")); checkpoint_params.set("Frequency",-1); + + // Create model restart OutputManager first. This OM will be in charge + // of creating rpointer.atm, while other OM's will simply append to it. + // If this assumption is not verified, we must always append to rpointer, which + // can make the rpointer file a bit confusing. if (io_params.isSublist("model_restart")) { - auto restart_pl = io_params.sublist("model_restart"); - restart_pl.set("Averaging Type","Instant"); - restart_pl.sublist("provenance") = m_atm_params.sublist("provenance"); - auto& om = m_output_managers.emplace_back(); - if (fvphyshack) { - // Don't save CGLL fields from ICs to the restart file. - std::map fms; - for (auto& it : m_field_mgrs) { - if (it.first == "Physics GLL") continue; - fms[it.first] = it.second; - } - om.set_logger(m_atm_logger); - om.setup(m_atm_comm,restart_pl, fms,m_grids_manager,m_run_t0,m_case_t0,true); - } else { - om.set_logger(m_atm_logger); - om.setup(m_atm_comm,restart_pl,m_field_mgrs,m_grids_manager,m_run_t0,m_case_t0,true); - } - om.set_logger(m_atm_logger); - for (const auto& it : m_atm_process_group->get_restart_extra_data()) { - om.add_global(it.first,it.second); - } + // Create model restart manager + auto params = io_params.sublist("model_restart"); + params.set("filename_prefix",m_casename+".scream"); + params.set("Averaging Type","Instant"); + params.sublist("provenance") = m_atm_params.sublist("provenance"); + + m_restart_output_manager = std::make_shared(); + m_restart_output_manager->initialize(m_atm_comm, + params, + m_run_t0, + m_case_t0, + /*is_model_restart_output*/ true); // Store the "Output Control" pl of the model restart as the "Checkpoint Control" for all other output streams - checkpoint_params.set("frequency_units",restart_pl.sublist("output_control").get("frequency_units")); - checkpoint_params.set("Frequency",restart_pl.sublist("output_control").get("Frequency")); + checkpoint_params.set("frequency_units",params.sublist("output_control").get("frequency_units")); + checkpoint_params.set("Frequency",params.sublist("output_control").get("Frequency")); } - // Build one manager per output yaml file + // Create one output manager per output yaml file using vos_t = std::vector; const auto& output_yaml_files = io_params.get("output_yaml_files",vos_t{}); - int om_tally = 0; for (const auto& fname : output_yaml_files) { ekat::ParameterList params; ekat::parse_yaml_file(fname,params); @@ -737,15 +802,59 @@ void AtmosphereDriver::initialize_output_managers () { // Check if the filename prefix for this file has already been set. If not, use the simulation casename. if (not params.isParameter("filename_prefix")) { - params.set("filename_prefix",m_casename+".scream.h"+std::to_string(om_tally)); - om_tally++; + params.set("filename_prefix",m_casename+".scream.h"); } params.sublist("provenance") = m_atm_params.sublist("provenance"); - // Add a new output manager - m_output_managers.emplace_back(); - auto& om = m_output_managers.back(); + + auto& om = m_output_managers.emplace_back(); + om.initialize(m_atm_comm, + params, + m_run_t0, + m_case_t0, + /*is_model_restart_output*/ false); + } + + m_ad_status |= s_output_created; + + stop_timer("EAMxx::create_output_managers"); + stop_timer("EAMxx::init"); + m_atm_logger->info("[EAMxx] create_output_managers ... done!"); +} + +void AtmosphereDriver::initialize_output_managers () { + m_atm_logger->info("[EAMxx] initialize_output_managers ..."); + start_timer("EAMxx::init"); + start_timer("EAMxx::initialize_output_managers"); + + check_ad_status (s_output_created | s_grids_created | s_fields_created); + + // Check for model restart output manager and setup if it exists. + if (m_restart_output_manager) { + if (fvphyshack) { + // Don't save CGLL fields from ICs to the restart file. + std::map fms; + for (auto& it : m_field_mgrs) { + if (it.first == "Physics GLL") continue; + fms[it.first] = it.second; + } + m_restart_output_manager->setup(fms, m_grids_manager); + } else { + m_restart_output_manager->setup(m_field_mgrs,m_grids_manager); + } + m_restart_output_manager->set_logger(m_atm_logger); + for (const auto& it : m_atm_process_group->get_restart_extra_data()) { + m_restart_output_manager->add_global(it.first,it.second); + } + } + + // Setup output managers + for (auto& om : m_output_managers) { + EKAT_REQUIRE_MSG(not om.is_restart(), + "Error! No restart output should be in m_output_managers. Model restart " + "output should be setup in m_restart_output_manager./n"); + om.set_logger(m_atm_logger); - om.setup(m_atm_comm,params,m_field_mgrs,m_grids_manager,m_run_t0,m_case_t0,false); + om.setup(m_field_mgrs,m_grids_manager); } m_ad_status |= s_output_inited; @@ -757,16 +866,21 @@ void AtmosphereDriver::initialize_output_managers () { void AtmosphereDriver:: set_provenance_data (std::string caseid, + std::string rest_caseid, std::string hostname, - std::string username) + std::string username, + std::string versionid) { #ifdef SCREAM_CIME_BUILD // Check the inputs are valid EKAT_REQUIRE_MSG (caseid!="", "Error! Invalid case id: " + caseid + "\n"); + EKAT_REQUIRE_MSG (m_run_type==RunType::Initial or rest_caseid!="", + "Error! Invalid restart case id: " + rest_caseid + "\n"); EKAT_REQUIRE_MSG (hostname!="", "Error! Invalid hostname: " + hostname + "\n"); EKAT_REQUIRE_MSG (username!="", "Error! Invalid username: " + username + "\n"); + EKAT_REQUIRE_MSG (versionid!="", "Error! Invalid version: " + versionid + "\n"); #else - caseid = "EAMxx standalone"; + caseid = rest_caseid = m_casename; char* user = new char[32]; char* host = new char[256]; int err; @@ -784,12 +898,14 @@ set_provenance_data (std::string caseid, } delete[] user; delete[] host; + versionid = EAMXX_GIT_VERSION; #endif auto& provenance = m_atm_params.sublist("provenance"); provenance.set("caseid",caseid); + provenance.set("rest_caseid",rest_caseid); provenance.set("hostname",hostname); provenance.set("username",username); - provenance.set("version",std::string(EAMXX_GIT_VERSION)); + provenance.set("git_version",versionid); } void AtmosphereDriver:: @@ -802,32 +918,12 @@ initialize_fields () start_timer("EAMxx::initialize_fields"); // See the [rrtmgp active gases] note in share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.hpp - if (fvphyshack) fv_phys_rrtmgp_active_gases_set_restart(m_case_t0 < m_run_t0); - - // See if we need to print a DAG. We do this first, cause if any input - // field is missing from the initial condition file, an error will be thrown. - // By printing the DAG first, we give the user the possibility of seeing - // what fields are inputs to the atm time step, so he/she can fix the i.c. file. - // TODO: would be nice to do the IC input first, and mark the fields in the - // DAG node "Begin of atm time step" in red if there's no initialization - // mechanism set for them. That is, allow field XYZ to not be found in - // the IC file, and throw an error when the dag is created. - - auto& driver_options_pl = m_atm_params.sublist("driver_options"); - const int verb_lvl = driver_options_pl.get("atmosphere_dag_verbosity_level",-1); - if (verb_lvl>0) { - // Check the atm DAG for missing stuff - AtmProcDAG dag; - - // First, add all atm processes - dag.create_dag(*m_atm_process_group); - - // Write a dot file for visualization - dag.write_dag("scream_atm_dag.dot",std::max(verb_lvl,0)); + if (fvphyshack) { + TraceGasesWorkaround::singleton().run_type = m_run_type; } // Initialize fields - if (m_case_t0info(" [EAMxx] restart_model ..."); // First, figure out the name of the netcdf file containing the restart data - const auto& casename = m_atm_params.sublist("initial_conditions").get("restart_casename"); - auto filename = find_filename_in_rpointer (casename,true,m_atm_comm,m_run_t0); + const auto& provenance = m_atm_params.sublist("provenance"); + const auto& casename = provenance.get("rest_caseid"); + auto filename = find_filename_in_rpointer (casename+".scream",true,m_atm_comm,m_run_t0); m_atm_logger->info(" [EAMxx] Restart filename: " + filename); @@ -1002,7 +1099,6 @@ void AtmosphereDriver::set_initial_conditions () // Check which fields need to have an initial condition. std::map> ic_fields_names; std::vector ic_fields_to_copy; - std::map> fields_inited; // Check which fields should be loaded from the topography file std::map> topography_file_fields_names; @@ -1031,7 +1127,7 @@ void AtmosphereDriver::set_initial_conditions () EKAT_ERROR_MSG ("ERROR: invalid assignment for variable " + fname + ", only scalar " "double or string, or vector double arguments are allowed"); } - fields_inited[grid_name].push_back(fname); + m_fields_inited[grid_name].push_back(fname); } else if (fname == "phis" or fname == "sgh30") { // Both phis and sgh30 need to be loaded from the topography file auto& this_grid_topo_file_fnames = topography_file_fields_names[grid_name]; @@ -1047,7 +1143,7 @@ void AtmosphereDriver::set_initial_conditions () grid_name == "Point Grid") { this_grid_topo_file_fnames.push_back("PHIS_d"); this_grid_topo_eamxx_fnames.push_back(fname); - fields_inited[grid_name].push_back(fname); + m_fields_inited[grid_name].push_back(fname); } else { EKAT_ERROR_MSG ("Error! Requesting phis on an unknown grid: " + grid_name + ".\n"); } @@ -1059,7 +1155,7 @@ void AtmosphereDriver::set_initial_conditions () " topo file only has sgh30 for Physics PG2.\n"); topography_file_fields_names[grid_name].push_back("SGH30"); topography_eamxx_fields_names[grid_name].push_back(fname); - fields_inited[grid_name].push_back(fname); + m_fields_inited[grid_name].push_back(fname); } } else if (not (fvphyshack and grid_name == "Physics PG2")) { // The IC file is written for the GLL grid, so we only load @@ -1071,7 +1167,7 @@ void AtmosphereDriver::set_initial_conditions () // If this field is the parent of other subfields, we only read from file the subfields. if (not ekat::contains(this_grid_ic_fnames,fname)) { this_grid_ic_fnames.push_back(fname); - fields_inited[grid_name].push_back(fname); + m_fields_inited[grid_name].push_back(fname); } } else if (fvphyshack and grid_name == "Physics GLL") { // [CGLL ICs in pg2] I tried doing something like this in @@ -1088,7 +1184,7 @@ void AtmosphereDriver::set_initial_conditions () } else { this_grid_ic_fnames.push_back(fname); } - fields_inited[grid_name].push_back(fname); + m_fields_inited[grid_name].push_back(fname); } } } @@ -1134,7 +1230,7 @@ void AtmosphereDriver::set_initial_conditions () auto p = f.get_header().get_parent().lock(); if (p) { const auto& pname = p->get_identifier().name(); - if (ekat::contains(fields_inited[grid_name],pname)) { + if (ekat::contains(m_fields_inited[grid_name],pname)) { // The parent is already inited. No need to init this field as well. names.erase(it2); run_again = true; @@ -1145,7 +1241,7 @@ void AtmosphereDriver::set_initial_conditions () } } - if (m_iop) { + if (m_iop_data_manager) { // For runs with IOP, call to setup io grids and lat // lon information needed for reading from file // We use a single topo file for both GLL and PG2 runs. All @@ -1155,13 +1251,13 @@ void AtmosphereDriver::set_initial_conditions () for (const auto& it : m_field_mgrs) { const auto& grid_name = it.first; if (ic_fields_names[grid_name].size() > 0 or - topography_eamxx_fields_names[grid_name].size() > 0) { + topography_eamxx_fields_names[grid_name].size() > 0) { const auto& file_name = grid_name == "Physics GLL" ? ic_pl.get("Filename") : ic_pl.get("topography_filename"); - m_iop->setup_io_info(file_name, it.second->get_grid()); + m_iop_data_manager->setup_io_info(file_name, it.second->get_grid()); } } } @@ -1173,12 +1269,12 @@ void AtmosphereDriver::set_initial_conditions () m_atm_logger->info(" [EAMxx] IC filename: " + file_name); for (const auto& it : m_field_mgrs) { const auto& grid_name = it.first; - if (not m_iop) { + if (not m_iop_data_manager) { read_fields_from_file (ic_fields_names[grid_name],it.second->get_grid(),file_name,m_current_ts); } else { // For IOP enabled, we load from file and copy data from the closest // lat/lon column to every other column - m_iop->read_fields_from_file_for_iop(file_name, + m_iop_data_manager->read_fields_from_file_for_iop(file_name, ic_fields_names[grid_name], m_current_ts, it.second); @@ -1248,7 +1344,7 @@ void AtmosphereDriver::set_initial_conditions () m_atm_logger->info(" filename: " + file_name); for (const auto& it : m_field_mgrs) { const auto& grid_name = it.first; - if (not m_iop) { + if (not m_iop_data_manager) { // Topography files always use "ncol_d" for the GLL grid value of ncol. // To ensure we read in the correct value, we must change the name for that dimension auto io_grid = it.second->get_grid(); @@ -1264,7 +1360,7 @@ void AtmosphereDriver::set_initial_conditions () } else { // For IOP enabled, we load from file and copy data from the closest // lat/lon column to every other column - m_iop->read_fields_from_file_for_iop(file_name, + m_iop_data_manager->read_fields_from_file_for_iop(file_name, topography_file_fields_names[grid_name], topography_eamxx_fields_names[grid_name], m_current_ts, @@ -1289,16 +1385,16 @@ void AtmosphereDriver::set_initial_conditions () m_atm_params.sublist("provenance").set("topography_file","NONE"); } - if (m_iop) { + if (m_iop_data_manager) { // Load IOP data file data for initial time stamp - m_iop->read_iop_file_data(m_current_ts); + m_iop_data_manager->read_iop_file_data(m_current_ts); // Now that ICs are processed, set appropriate fields using IOP file data. // Since ICs are loaded on GLL grid, we set those fields only and dynamics // will take care of the rest (for PG2 case). if (m_field_mgrs.count("Physics GLL") > 0) { const auto& fm = m_field_mgrs.at("Physics GLL"); - m_iop->set_fields_from_iop_data(fm); + m_iop_data_manager->set_fields_from_iop_data(fm); } } @@ -1357,7 +1453,7 @@ void AtmosphereDriver::set_initial_conditions () // Loop through fields and apply perturbation. for (size_t f=0; fget_grid()->name()], fname), + EKAT_REQUIRE_MSG(ekat::contains(m_fields_inited[fm->get_grid()->name()], fname), "Error! Attempting to apply perturbation to field not in initial_conditions.\n" " - Field: "+fname+"\n" " - Grid: "+fm->get_grid()->name()+"\n"); @@ -1523,15 +1619,13 @@ void AtmosphereDriver::initialize_atm_procs () m_memory_buffer->allocate(); m_atm_process_group->init_buffers(*m_memory_buffer); - const bool restarted_run = m_case_t0 < m_run_t0; - // Setup SurfaceCoupling import and export (if they exist) if (m_surface_coupling_import_data_manager || m_surface_coupling_export_data_manager) { setup_surface_coupling_processes(); } // Initialize the processes - m_atm_process_group->initialize(m_current_ts, restarted_run ? RunType::Restarted : RunType::Initial); + m_atm_process_group->initialize(m_current_ts, m_run_type); // Create and add energy and mass conservation check to appropriate atm procs setup_column_conservation_checks(); @@ -1558,6 +1652,28 @@ void AtmosphereDriver::initialize_atm_procs () m_atm_logger->info("[EAMxx] initialize_atm_procs ... done!"); report_res_dep_memory_footprint (); + + auto& driver_options_pl = m_atm_params.sublist("driver_options"); + const int verb_lvl = driver_options_pl.get("atmosphere_dag_verbosity_level",-1); + if (verb_lvl>0) { + // now that we've got fields, generate a DAG with fields and dependencies + // NOTE: at this point, fields provided by initial conditions may (will) + // appear as unmet dependencies + AtmProcDAG dag; + // First, add all atm processes + dag.create_dag(*m_atm_process_group); + // process the initial conditions to maybe fulfill unmet dependencies + dag.process_initial_conditions(m_fields_inited); + // Write a dot file for visualizing the DAG + if (m_atm_comm.am_i_root()) { + std::string filename = "scream_atm_initProc_dag"; + if (is_scream_standalone()) { + filename += ".np" + std::to_string(m_atm_comm.size()); + } + filename += ".dot"; + dag.write_dag(filename, verb_lvl); + } + } } void AtmosphereDriver:: @@ -1574,6 +1690,8 @@ initialize (const ekat::Comm& atm_comm, init_time_stamps (run_t0, case_t0); + create_output_managers (); + create_atm_processes (); create_grids (); @@ -1595,10 +1713,10 @@ void AtmosphereDriver::run (const int dt) { start_timer("EAMxx::run"); // DEBUG option: Check if user has set the run to fail at a specific timestep. - auto& debug = m_atm_params.sublist("driver_debug_options"); - auto fail_step = debug.get("force_crash_nsteps",-1); - if (fail_step==m_current_ts.get_num_steps()) { - std::abort(); + auto& debug = m_atm_params.sublist("driver_debug_options"); + auto fail_step = debug.get("force_crash_nsteps",-1); + if (fail_step==m_current_ts.get_num_steps()) { + std::abort(); } // Make sure the end of the time step is after the current start_time @@ -1622,6 +1740,7 @@ void AtmosphereDriver::run (const int dt) { // that quantity at the beginning of the timestep. Or they may need to store // the timestamp at the beginning of the timestep, so that we can compute // dt at the end. + if (m_restart_output_manager) m_restart_output_manager->init_timestep(m_current_ts, dt); for (auto& it : m_output_managers) { it.init_timestep(m_current_ts,dt); } @@ -1648,6 +1767,7 @@ void AtmosphereDriver::run (const int dt) { // Update output streams m_atm_logger->debug("[EAMxx::run] running output managers..."); + if (m_restart_output_manager) m_restart_output_manager->run(m_current_ts); for (auto& out_mgr : m_output_managers) { out_mgr.run(m_current_ts); } @@ -1678,6 +1798,10 @@ void AtmosphereDriver::finalize ( /* inputs? */ ) { m_atm_logger->info("[EAMxx] Finalize ..."); // Finalize and destroy output streams, make sure files are closed + if (m_restart_output_manager) { + m_restart_output_manager->finalize(); + m_restart_output_manager = nullptr; + } for (auto& out_mgr : m_output_managers) { out_mgr.finalize(); } @@ -1690,7 +1814,7 @@ void AtmosphereDriver::finalize ( /* inputs? */ ) { } // Destroy iop - m_iop = nullptr; + m_iop_data_manager = nullptr; // Destroy the buffer manager m_memory_buffer = nullptr; @@ -1801,6 +1925,11 @@ void AtmosphereDriver::report_res_dep_memory_footprint () const { // Atm buffer my_dev_mem_usage += m_memory_buffer->allocated_bytes(); // Output + if (m_restart_output_manager) { + const auto om_footprint = m_restart_output_manager->res_dep_memory_footprint(); + my_dev_mem_usage += om_footprint; + my_host_mem_usage += om_footprint; + } for (const auto& om : m_output_managers) { const auto om_footprint = om.res_dep_memory_footprint (); my_dev_mem_usage += om_footprint; diff --git a/components/eamxx/src/control/atmosphere_driver.hpp b/components/eamxx/src/control/atmosphere_driver.hpp index 7e96225210db..408b82437e44 100644 --- a/components/eamxx/src/control/atmosphere_driver.hpp +++ b/components/eamxx/src/control/atmosphere_driver.hpp @@ -2,7 +2,6 @@ #define SCREAM_ATMOSPHERE_DRIVER_HPP #include "control/surface_coupling_utils.hpp" -#include "share/iop/intensive_observation_period.hpp" #include "share/field/field_manager.hpp" #include "share/grid/grids_manager.hpp" #include "share/util/scream_time_stamp.hpp" @@ -11,6 +10,7 @@ #include "share/io/scorpio_input.hpp" #include "share/atm_process/ATMBufferManager.hpp" #include "share/atm_process/SCDataManager.hpp" +#include "share/atm_process/IOPDataManager.hpp" #include "ekat/logging/ekat_logger.hpp" #include "ekat/mpi/ekat_comm.hpp" @@ -66,13 +66,14 @@ class AtmosphereDriver void set_params (const ekat::ParameterList& params); // Init time stamps - void init_time_stamps (const util::TimeStamp& run_t0, const util::TimeStamp& case_t0); + // run_type: -1: deduce from run/case t0, 0: initial, 1: restart + void init_time_stamps (const util::TimeStamp& run_t0, const util::TimeStamp& case_t0, int run_type = -1); // Set AD params void init_scorpio (const int atm_id = 0); - // Setup IntensiveObservationPeriod - void setup_iop (); + // Setup IOPDataManager + void setup_iop_data_manager (); // Create atm processes, without initializing them void create_atm_processes (); @@ -113,12 +114,17 @@ class AtmosphereDriver void add_additional_column_data_to_property_checks (); void set_provenance_data (std::string caseid = "", + std::string rest_caseid = "", std::string hostname = "", - std::string username = ""); + std::string username = "", + std::string versionid = ""); // Load initial conditions for atm inputs void initialize_fields (); + // Create output managers + void create_output_managers (); + // Initialie I/O structures for output void initialize_output_managers (); @@ -204,13 +210,14 @@ class AtmosphereDriver ekat::ParameterList m_atm_params; + std::shared_ptr m_restart_output_manager; std::list m_output_managers; std::shared_ptr m_memory_buffer; std::shared_ptr m_surface_coupling_import_data_manager; std::shared_ptr m_surface_coupling_export_data_manager; - std::shared_ptr m_iop; + std::shared_ptr m_iop_data_manager; // This is the time stamp at the beginning of the time step. util::TimeStamp m_current_ts; @@ -220,6 +227,7 @@ class AtmosphereDriver // restarted runs, the latter is "older" than the former util::TimeStamp m_run_t0; util::TimeStamp m_case_t0; + RunType m_run_type; // This is the comm containing all (and only) the processes assigned to the atmosphere ekat::Comm m_atm_comm; @@ -239,6 +247,7 @@ class AtmosphereDriver static constexpr int s_fields_inited = 256; static constexpr int s_procs_inited = 512; static constexpr int s_ts_inited = 1024; + static constexpr int s_output_created = 2048; // Lazy version to ensure s_atm_inited & flag is true for every flag, // even if someone adds new flags later on @@ -255,6 +264,8 @@ class AtmosphereDriver // Current simulation casename std::string m_casename; + // maps grid name to a vector of its initialized fields + std::map> m_fields_inited; }; } // namespace control diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_exporter.cpp b/components/eamxx/src/control/atmosphere_surface_coupling_exporter.cpp index 9044df7e76cc..5b5ea600a857 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_exporter.cpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_exporter.cpp @@ -43,8 +43,8 @@ void SurfaceCouplingExporter::set_grids(const std::shared_ptr("pseudo_density", scalar3d_layout_mid, Pa, grid_name, ps); add_field("phis", scalar2d_layout, m2/s2, grid_name); add_field("p_mid", scalar3d_layout_mid, Pa, grid_name, ps); - add_field("qv", scalar3d_layout_mid, kg/kg, grid_name, "tracers", ps); add_field("T_mid", scalar3d_layout_mid, K, grid_name, ps); + add_tracer("qv", m_grid, kg/kg, ps); // TODO: Switch horiz_winds to using U and V, note right now there is an issue with when the subfields are created, so can't switch yet. add_field("horiz_winds", vector3d_layout, m/s, grid_name); add_field("sfc_flux_dir_nir", scalar2d_layout, W/m2, grid_name); diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp index ee3e21e7461b..385e57cae556 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp @@ -28,35 +28,41 @@ void SurfaceCouplingImporter::set_grids(const std::shared_ptr("sfc_alb_dir_vis", scalar2d_layout, nondim, grid_name); - add_field("sfc_alb_dir_nir", scalar2d_layout, nondim, grid_name); - add_field("sfc_alb_dif_vis", scalar2d_layout, nondim, grid_name); - add_field("sfc_alb_dif_nir", scalar2d_layout, nondim, grid_name); - add_field("surf_lw_flux_up", scalar2d_layout, W/m2, grid_name); - add_field("surf_sens_flux", scalar2d_layout, W/m2, grid_name); - add_field("surf_evap", scalar2d_layout, kg/m2/s, grid_name); - add_field("surf_mom_flux", vector2d_layout, N/m2, grid_name); - add_field("surf_radiative_T", scalar2d_layout, K, grid_name); - add_field("T_2m", scalar2d_layout, K, grid_name); - add_field("qv_2m", scalar2d_layout, kg/kg, grid_name); - add_field("wind_speed_10m", scalar2d_layout, m/s, grid_name); - add_field("snow_depth_land", scalar2d_layout, m, grid_name); - add_field("ocnfrac", scalar2d_layout, nondim, grid_name); - add_field("landfrac", scalar2d_layout, nondim, grid_name); - add_field("icefrac", scalar2d_layout, nondim, grid_name); + const FieldLayout scalar2d = m_grid->get_2d_scalar_layout(); + const FieldLayout vector2d = m_grid->get_2d_vector_layout(2); + const FieldLayout vector4d = m_grid->get_2d_vector_layout(4); + + add_field("sfc_alb_dir_vis", scalar2d, nondim, grid_name); + add_field("sfc_alb_dir_nir", scalar2d, nondim, grid_name); + add_field("sfc_alb_dif_vis", scalar2d, nondim, grid_name); + add_field("sfc_alb_dif_nir", scalar2d, nondim, grid_name); + add_field("surf_lw_flux_up", scalar2d, W/m2, grid_name); + add_field("surf_sens_flux", scalar2d, W/m2, grid_name); + add_field("surf_evap", scalar2d, kg/m2/s, grid_name); + add_field("surf_mom_flux", vector2d, N/m2, grid_name); + add_field("surf_radiative_T", scalar2d, K, grid_name); + add_field("T_2m", scalar2d, K, grid_name); + add_field("qv_2m", scalar2d, kg/kg, grid_name); + add_field("wind_speed_10m", scalar2d, m/s, grid_name); + add_field("snow_depth_land", scalar2d, m, grid_name); + add_field("ocnfrac", scalar2d, nondim, grid_name); + add_field("landfrac", scalar2d, nondim, grid_name); + add_field("icefrac", scalar2d, nondim, grid_name); // Friction velocity [m/s] - add_field("fv", scalar2d_layout, m/s, grid_name); + add_field("fv", scalar2d, m/s, grid_name); // Aerodynamical resistance - add_field("ram1", scalar2d_layout, s/m, grid_name); + add_field("ram1", scalar2d, s/m, grid_name); + // Sea surface temperature [K] + add_field("sst", scalar2d, K, grid_name); + //dust fluxes [kg/m^2/s]: Four flux values for eacch column + add_field("dstflx", vector4d, kg/m2/s, grid_name); + } // ========================================================================================= void SurfaceCouplingImporter::setup_surface_coupling_data(const SCDataManager &sc_data_manager) @@ -202,8 +208,8 @@ void SurfaceCouplingImporter::do_import(const bool called_during_initialization) }); #endif - if (m_iop) { - if (m_iop->get_params().get("iop_srf_prop")) { + if (m_iop_data_manager) { + if (m_iop_data_manager->get_params().get("iop_srf_prop")) { // Overwrite imports with data from IOP file overwrite_iop_imports(called_during_initialization); } @@ -215,9 +221,12 @@ void SurfaceCouplingImporter::overwrite_iop_imports (const bool called_during_in using policy_type = KokkosTypes::RangePolicy; using C = physics::Constants; - const auto has_lhflx = m_iop->has_iop_field("lhflx"); - const auto has_shflx = m_iop->has_iop_field("shflx"); - const auto has_Tg = m_iop->has_iop_field("Tg"); + const auto has_lhflx = m_iop_data_manager->has_iop_field("lhflx"); + const auto has_shflx = m_iop_data_manager->has_iop_field("shflx"); + const auto has_Tg = m_iop_data_manager->has_iop_field("Tg"); + + // Read IOP file for current time step, if necessary + m_iop_data_manager->read_iop_file_data(timestamp()); static constexpr Real latvap = C::LatVap; static constexpr Real stebol = C::stebol; @@ -237,19 +246,19 @@ void SurfaceCouplingImporter::overwrite_iop_imports (const bool called_during_in // Store IOP surf data into col_val Real col_val(std::nan("")); if (fname == "surf_evap" && has_lhflx) { - const auto f = m_iop->get_iop_field("lhflx"); + const auto f = m_iop_data_manager->get_iop_field("lhflx"); f.sync_to_host(); col_val = f.get_view()()/latvap; } else if (fname == "surf_sens_flux" && has_shflx) { - const auto f = m_iop->get_iop_field("shflx"); + const auto f = m_iop_data_manager->get_iop_field("shflx"); f.sync_to_host(); col_val = f.get_view()(); } else if (fname == "surf_radiative_T" && has_Tg) { - const auto f = m_iop->get_iop_field("Tg"); + const auto f = m_iop_data_manager->get_iop_field("Tg"); f.sync_to_host(); col_val = f.get_view()(); } else if (fname == "surf_lw_flux_up" && has_Tg) { - const auto f = m_iop->get_iop_field("Tg"); + const auto f = m_iop_data_manager->get_iop_field("Tg"); f.sync_to_host(); col_val = stebol*std::pow(f.get_view()(), 4); } else { diff --git a/components/eamxx/src/diagnostics/CMakeLists.txt b/components/eamxx/src/diagnostics/CMakeLists.txt index be51f4346155..0795f9e54694 100644 --- a/components/eamxx/src/diagnostics/CMakeLists.txt +++ b/components/eamxx/src/diagnostics/CMakeLists.txt @@ -1,11 +1,16 @@ set(DIAGNOSTIC_SRCS + aerocom_cld.cpp + aodvis.cpp + atm_backtend.cpp atm_density.cpp dry_static_energy.cpp exner.cpp field_at_height.cpp field_at_level.cpp field_at_pressure_level.cpp + horiz_avg.cpp longwave_cloud_forcing.cpp + number_path.cpp potential_temperature.cpp precip_surf_mass_flux.cpp relative_humidity.cpp @@ -17,15 +22,11 @@ set(DIAGNOSTIC_SRCS virtual_temperature.cpp water_path.cpp wind_speed.cpp - aodvis.cpp - number_path.cpp - aerocom_cld.cpp - atm_backtend.cpp ) add_library(diagnostics ${DIAGNOSTIC_SRCS}) target_link_libraries(diagnostics PUBLIC scream_share) -if (NOT SCREAM_LIB_ONLY) +if (NOT SCREAM_LIB_ONLY AND NOT SCREAM_ONLY_GENERATE_BASELINES) add_subdirectory(tests) endif() diff --git a/components/eamxx/src/diagnostics/aerocom_cld.cpp b/components/eamxx/src/diagnostics/aerocom_cld.cpp index 8606e065b9e3..cca55f6c19f1 100644 --- a/components/eamxx/src/diagnostics/aerocom_cld.cpp +++ b/components/eamxx/src/diagnostics/aerocom_cld.cpp @@ -22,8 +22,6 @@ AeroComCld::AeroComCld(const ekat::Comm &comm, "to be 'Bot' or 'Top' in its input parameters.\n"); } -std::string AeroComCld::name() const { return "AeroComCld" + m_topbot; } - void AeroComCld::set_grids( const std::shared_ptr grids_manager) { using namespace ekat::units; @@ -76,7 +74,8 @@ void AeroComCld::set_grids( m_dz.allocate_view(); // Construct and allocate the output field - FieldIdentifier fid(name(), vector1d_layout, nondim, grid_name); + + FieldIdentifier fid("AeroComCld"+m_topbot, vector1d_layout, nondim, grid_name); m_diagnostic_output = Field(fid); m_diagnostic_output.allocate_view(); diff --git a/components/eamxx/src/diagnostics/aerocom_cld.hpp b/components/eamxx/src/diagnostics/aerocom_cld.hpp index 694eed76f097..53a9a7f8e1ba 100644 --- a/components/eamxx/src/diagnostics/aerocom_cld.hpp +++ b/components/eamxx/src/diagnostics/aerocom_cld.hpp @@ -15,7 +15,7 @@ class AeroComCld : public AtmosphereDiagnostic { AeroComCld(const ekat::Comm &comm, const ekat::ParameterList ¶ms); // The name of the diagnostic - std::string name() const override; + std::string name() const override { return "AeroComCld"; } // Set the grid void set_grids( diff --git a/components/eamxx/src/diagnostics/field_at_height.cpp b/components/eamxx/src/diagnostics/field_at_height.cpp index f61cd3a76c1f..db780f5a79fd 100644 --- a/components/eamxx/src/diagnostics/field_at_height.cpp +++ b/components/eamxx/src/diagnostics/field_at_height.cpp @@ -45,21 +45,16 @@ FieldAtHeight (const ekat::Comm& comm, const ekat::ParameterList& params) " - surface reference: " + surf_ref + "\n" " - valid options: sealevel, surface\n"); m_z_name = (surf_ref == "sealevel") ? "z" : "height"; - const auto& location = m_params.get("vertical_location"); - auto chars_start = location.find_first_not_of("0123456789."); - EKAT_REQUIRE_MSG (chars_start!=0 && chars_start!=std::string::npos, - "Error! Invalid string for height value for FieldAtHeight.\n" - " - input string : " + location + "\n" - " - expected format: Nm, with N integer\n"); - const auto z_str = location.substr(0,chars_start); - m_z = std::stod(z_str); - - const auto units = location.substr(chars_start); + + const auto units = m_params.get("height_units"); EKAT_REQUIRE_MSG (units=="m", - "Error! Invalid string for height value for FieldAtHeight.\n" - " - input string : " + location + "\n" - " - expected format: Nm, with N integer\n"); - m_diag_name = m_field_name + "_at_" + m_params.get("vertical_location") + "_above_" + surf_ref; + "Error! Invalid units for FieldAtHeight.\n" + " - input units: " + units + "\n" + " - valid units: m\n"); + + auto z_val = m_params.get("height_value"); + m_z = std::stod(z_val); + m_diag_name = m_field_name + "_at_" + z_val + units + "_above_" + surf_ref; } void FieldAtHeight:: @@ -89,7 +84,9 @@ initialize_impl (const RunType /*run_type*/) EKAT_REQUIRE_MSG (layout.rank()>=2 && layout.rank()<=3, "Error! Field rank not supported by FieldAtHeight.\n" " - field name: " + fid.name() + "\n" - " - field layout: " + layout.to_string() + "\n"); + " - field layout: " + layout.to_string() + "\n" + "NOTE: if you requested something like 'field_horiz_avg_at_Y',\n" + " you can avoid this error by requesting 'fieldX_at_Y_horiz_avg' instead.\n"); const auto tag = layout.tags().back(); EKAT_REQUIRE_MSG (tag==LEV || tag==ILEV, "Error! FieldAtHeight diagnostic expects a layout ending with 'LEV'/'ILEV' tag.\n" diff --git a/components/eamxx/src/diagnostics/field_at_height.hpp b/components/eamxx/src/diagnostics/field_at_height.hpp index e6198153f940..91f6ae3eb1b5 100644 --- a/components/eamxx/src/diagnostics/field_at_height.hpp +++ b/components/eamxx/src/diagnostics/field_at_height.hpp @@ -18,7 +18,7 @@ class FieldAtHeight : public AtmosphereDiagnostic FieldAtHeight (const ekat::Comm& comm, const ekat::ParameterList& params); // The name of the diagnostic - std::string name () const { return m_diag_name; } + std::string name () const { return "FieldAtHeight"; } // Set the grid void set_grids (const std::shared_ptr grids_manager); diff --git a/components/eamxx/src/diagnostics/field_at_level.cpp b/components/eamxx/src/diagnostics/field_at_level.cpp index 87ecf7ad9103..b842429a8d6e 100644 --- a/components/eamxx/src/diagnostics/field_at_level.cpp +++ b/components/eamxx/src/diagnostics/field_at_level.cpp @@ -30,10 +30,12 @@ initialize_impl (const RunType /*run_type*/) using namespace ShortFieldTagsNames; const auto& fid = f.get_header().get_identifier(); const auto& layout = fid.get_layout(); - EKAT_REQUIRE_MSG (layout.rank()>1 && layout.rank()<=6, + EKAT_REQUIRE_MSG (layout.rank()>=2 && layout.rank()<=6, "Error! Field rank not supported by FieldAtLevel.\n" " - field name: " + fid.name() + "\n" - " - field layout: " + layout.to_string() + "\n"); + " - field layout: " + layout.to_string() + "\n" + "NOTE: if you requested something like 'field_horiz_avg_at_Y',\n" + " you can avoid this error by requesting 'fieldX_at_Y_horiz_avg' instead.\n"); const auto tag = layout.tags().back(); EKAT_REQUIRE_MSG (tag==LEV || tag==ILEV, "Error! FieldAtLevel diagnostic expects a layout ending with 'LEV'/'ILEV' tag.\n" diff --git a/components/eamxx/src/diagnostics/field_at_level.hpp b/components/eamxx/src/diagnostics/field_at_level.hpp index b63cda0a1a38..3ab9bee15574 100644 --- a/components/eamxx/src/diagnostics/field_at_level.hpp +++ b/components/eamxx/src/diagnostics/field_at_level.hpp @@ -21,7 +21,7 @@ class FieldAtLevel : public AtmosphereDiagnostic FieldAtLevel (const ekat::Comm& comm, const ekat::ParameterList& params); // The name of the diagnostic - std::string name () const { return m_diag_name; } + std::string name () const { return "FieldAtLevel"; } // Set the grid void set_grids (const std::shared_ptr grids_manager); diff --git a/components/eamxx/src/diagnostics/field_at_pressure_level.cpp b/components/eamxx/src/diagnostics/field_at_pressure_level.cpp index 21c1ac78dd90..78ed921e7585 100644 --- a/components/eamxx/src/diagnostics/field_at_pressure_level.cpp +++ b/components/eamxx/src/diagnostics/field_at_pressure_level.cpp @@ -15,30 +15,22 @@ FieldAtPressureLevel (const ekat::Comm& comm, const ekat::ParameterList& params) { m_field_name = m_params.get("field_name"); - // Figure out the pressure value - const auto& location = m_params.get("vertical_location"); - auto chars_start = location.find_first_not_of("0123456789."); - EKAT_REQUIRE_MSG (chars_start!=0 && chars_start!=std::string::npos, - "Error! Invalid string for pressure value for FieldAtPressureLevel.\n" - " - input string : " + location + "\n" - " - expected format: Nxyz, with N integer, and xyz='mb', 'hPa', or 'Pa'\n"); - const auto press_str = location.substr(0,chars_start); - m_pressure_level = std::stod(press_str); - - const auto units = location.substr(chars_start); + const auto units = m_params.get("pressure_units"); EKAT_REQUIRE_MSG (units=="mb" or units=="hPa" or units=="Pa", - "Error! Invalid string for pressure value for FieldAtPressureLevel.\n" - " - input string : " + location + "\n" - " - expected format: Nxyz, with N integer, and xyz='mb', 'hPa', or 'Pa'\n"); + "Error! Invalid units for FieldAtPressureLevel.\n" + " - input units: " + units + "\n" + " - valid units: 'mb', 'hPa', 'Pa'\n"); + + // Figure out the pressure value, and convert to Pa if needed + auto p_value = m_params.get("pressure_value"); - // Convert pressure level to Pa, the units of pressure in the simulation if (units=="mb" || units=="hPa") { - m_pressure_level *= 100; + m_pressure_level = std::stod(p_value)*100; + } else { + m_pressure_level = std::stod(p_value); } - m_mask_val = m_params.get("mask_value",Real(constants::DefaultFillValue::value)); - - m_diag_name = m_field_name + "_at_" + location; + m_diag_name = m_field_name + "_at_" + p_value + units; } void FieldAtPressureLevel:: @@ -64,7 +56,9 @@ initialize_impl (const RunType /*run_type*/) EKAT_REQUIRE_MSG (layout.rank()>=2 && layout.rank()<=3, "Error! Field rank not supported by FieldAtPressureLevel.\n" " - field name: " + fid.name() + "\n" - " - field layout: " + layout.to_string() + "\n"); + " - field layout: " + layout.to_string() + "\n" + "NOTE: if you requested something like 'field_horiz_avg_at_Y',\n" + " you can avoid this error by requesting 'fieldX_at_Y_horiz_avg' instead.\n"); const auto tag = layout.tags().back(); EKAT_REQUIRE_MSG (tag==LEV || tag==ILEV, "Error! FieldAtPressureLevel diagnostic expects a layout ending with 'LEV'/'ILEV' tag.\n" @@ -91,6 +85,8 @@ initialize_impl (const RunType /*run_type*/) // Add a field representing the mask as extra data to the diagnostic field. auto nondim = ekat::units::Units::nondimensional(); const auto& gname = fid.get_grid_name(); + m_mask_val = m_params.get("mask_value",Real(constants::DefaultFillValue::value)); + std::string mask_name = name() + " mask"; FieldLayout mask_layout( {COL}, {num_cols}); diff --git a/components/eamxx/src/diagnostics/field_at_pressure_level.hpp b/components/eamxx/src/diagnostics/field_at_pressure_level.hpp index 950c0c5e2ee9..58e476ec83b9 100644 --- a/components/eamxx/src/diagnostics/field_at_pressure_level.hpp +++ b/components/eamxx/src/diagnostics/field_at_pressure_level.hpp @@ -20,7 +20,7 @@ class FieldAtPressureLevel : public AtmosphereDiagnostic FieldAtPressureLevel (const ekat::Comm& comm, const ekat::ParameterList& params); // The name of the diagnostic - std::string name () const { return m_diag_name; } + std::string name () const { return "FieldAtPressureLevel"; } // Set the grid void set_grids (const std::shared_ptr grids_manager); diff --git a/components/eamxx/src/diagnostics/horiz_avg.cpp b/components/eamxx/src/diagnostics/horiz_avg.cpp new file mode 100644 index 000000000000..8bfe068c635b --- /dev/null +++ b/components/eamxx/src/diagnostics/horiz_avg.cpp @@ -0,0 +1,65 @@ +#include "diagnostics/horiz_avg.hpp" + +#include "share/field/field_utils.hpp" + +namespace scream { + +HorizAvgDiag::HorizAvgDiag(const ekat::Comm &comm, + const ekat::ParameterList ¶ms) + : AtmosphereDiagnostic(comm, params) { + const auto &fname = m_params.get("field_name"); + m_diag_name = fname + "_horiz_avg"; +} + +void HorizAvgDiag::set_grids( + const std::shared_ptr grids_manager) { + const auto &fn = m_params.get("field_name"); + const auto &gn = m_params.get("grid_name"); + const auto g = grids_manager->get_grid("Physics"); + + add_field(fn, gn); + + // first clone the area unscaled, we will scale it later in initialize_impl + m_scaled_area = g->get_geometry_data("area").clone(); +} + +void HorizAvgDiag::initialize_impl(const RunType /*run_type*/) { + using namespace ShortFieldTagsNames; + const auto &f = get_fields_in().front(); + const auto &fid = f.get_header().get_identifier(); + const auto &layout = fid.get_layout(); + + EKAT_REQUIRE_MSG(layout.rank() >= 1 && layout.rank() <= 3, + "Error! Field rank not supported by HorizAvgDiag.\n" + " - field name: " + + fid.name() + + "\n" + " - field layout: " + + layout.to_string() + "\n"); + EKAT_REQUIRE_MSG(layout.tags()[0] == COL, + "Error! HorizAvgDiag diagnostic expects a layout starting " + "with the 'COL' tag.\n" + " - field name : " + + fid.name() + + "\n" + " - field layout: " + + layout.to_string() + "\n"); + + FieldIdentifier d_fid(m_diag_name, layout.clone().strip_dim(COL), + fid.get_units(), fid.get_grid_name()); + m_diagnostic_output = Field(d_fid); + m_diagnostic_output.allocate_view(); + + // scale the area field + auto total_area = field_sum(m_scaled_area, &m_comm); + m_scaled_area.scale(sp(1.0) / total_area); +} + +void HorizAvgDiag::compute_diagnostic_impl() { + const auto &f = get_fields_in().front(); + const auto &d = m_diagnostic_output; + // Call the horiz_contraction impl that will take care of everything + horiz_contraction(d, f, m_scaled_area, &m_comm); +} + +} // namespace scream diff --git a/components/eamxx/src/diagnostics/horiz_avg.hpp b/components/eamxx/src/diagnostics/horiz_avg.hpp new file mode 100644 index 000000000000..6ceac09103be --- /dev/null +++ b/components/eamxx/src/diagnostics/horiz_avg.hpp @@ -0,0 +1,43 @@ +#ifndef EAMXX_HORIZ_AVERAGE_HPP +#define EAMXX_HORIZ_AVERAGE_HPP + +#include "share/atm_process/atmosphere_diagnostic.hpp" + +namespace scream { + +/* + * This diagnostic will calculate the area-weighted average of a field + * across the COL tag dimension, producing an N-1 dimensional field + * that is area-weighted average of the input field. + */ + +class HorizAvgDiag : public AtmosphereDiagnostic { + public: + // Constructors + HorizAvgDiag(const ekat::Comm &comm, const ekat::ParameterList ¶ms); + + // The name of the diagnostic + std::string name() const { return m_diag_name; } + + // Set the grid + void set_grids(const std::shared_ptr grids_manager); + + protected: +#ifdef KOKKOS_ENABLE_CUDA + public: +#endif + void compute_diagnostic_impl(); + + protected: + void initialize_impl(const RunType /*run_type*/); + + // Name of each field (because the diagnostic impl is generic) + std::string m_diag_name; + + // Need area field, let's store it scaled by its norm + Field m_scaled_area; +}; + +} // namespace scream + +#endif // EAMXX_HORIZ_AVERAGE_HPP diff --git a/components/eamxx/src/diagnostics/number_path.cpp b/components/eamxx/src/diagnostics/number_path.cpp index d4df2f1bc22f..70e18c6dee30 100644 --- a/components/eamxx/src/diagnostics/number_path.cpp +++ b/components/eamxx/src/diagnostics/number_path.cpp @@ -33,8 +33,6 @@ NumberPathDiagnostic::NumberPathDiagnostic(const ekat::Comm &comm, } } -std::string NumberPathDiagnostic::name() const { return m_kind + "NumberPath"; } - void NumberPathDiagnostic::set_grids( const std::shared_ptr grids_manager) { using namespace ekat::units; @@ -55,7 +53,7 @@ void NumberPathDiagnostic::set_grids( add_field(m_nname, scalar3d, 1 / kg, grid_name); // Construct and allocate the diagnostic field - FieldIdentifier fid(name(), scalar2d, kg/(kg*m2), grid_name); + FieldIdentifier fid(m_kind + "NumberPath", scalar2d, kg/(kg*m2), grid_name); m_diagnostic_output = Field(fid); m_diagnostic_output.allocate_view(); } diff --git a/components/eamxx/src/diagnostics/number_path.hpp b/components/eamxx/src/diagnostics/number_path.hpp index 4888d3601f44..30b383b9452f 100644 --- a/components/eamxx/src/diagnostics/number_path.hpp +++ b/components/eamxx/src/diagnostics/number_path.hpp @@ -16,7 +16,7 @@ class NumberPathDiagnostic : public AtmosphereDiagnostic { const ekat::ParameterList ¶ms); // The name of the diagnostic - std::string name() const; + std::string name() const override { return "NumberPath"; } // Set the grid void set_grids(const std::shared_ptr grids_manager); diff --git a/components/eamxx/src/diagnostics/potential_temperature.cpp b/components/eamxx/src/diagnostics/potential_temperature.cpp index 67260e647f6f..8cc45078de76 100644 --- a/components/eamxx/src/diagnostics/potential_temperature.cpp +++ b/components/eamxx/src/diagnostics/potential_temperature.cpp @@ -24,11 +24,6 @@ PotentialTemperatureDiagnostic::PotentialTemperatureDiagnostic (const ekat::Comm } } -std::string PotentialTemperatureDiagnostic::name() const -{ - return m_ptype; -} - // ========================================================================================= void PotentialTemperatureDiagnostic::set_grids(const std::shared_ptr grids_manager) { @@ -51,7 +46,7 @@ void PotentialTemperatureDiagnostic::set_grids(const std::shared_ptr("qc", scalar3d_layout_mid, kg/kg, grid_name, ps); // Construct and allocate the diagnostic field - FieldIdentifier fid (name(), scalar3d_layout_mid, K, grid_name); + FieldIdentifier fid (m_ptype, scalar3d_layout_mid, K, grid_name); m_diagnostic_output = Field(fid); auto& C_ap = m_diagnostic_output.get_header().get_alloc_properties(); C_ap.request_allocation(ps); diff --git a/components/eamxx/src/diagnostics/potential_temperature.hpp b/components/eamxx/src/diagnostics/potential_temperature.hpp index 37fd1a30806c..0ac1a1201d8f 100644 --- a/components/eamxx/src/diagnostics/potential_temperature.hpp +++ b/components/eamxx/src/diagnostics/potential_temperature.hpp @@ -22,7 +22,7 @@ class PotentialTemperatureDiagnostic : public AtmosphereDiagnostic PotentialTemperatureDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); // The name of the diagnostic - std::string name () const; + std::string name () const override { return "PotentialTemperature"; } // Set the grid void set_grids (const std::shared_ptr grids_manager); diff --git a/components/eamxx/src/diagnostics/precip_surf_mass_flux.cpp b/components/eamxx/src/diagnostics/precip_surf_mass_flux.cpp index 068456f0522d..329e83b6e9cc 100644 --- a/components/eamxx/src/diagnostics/precip_surf_mass_flux.cpp +++ b/components/eamxx/src/diagnostics/precip_surf_mass_flux.cpp @@ -47,7 +47,7 @@ set_grids(const std::shared_ptr grids_manager) } // Construct and allocate the diagnostic field - FieldIdentifier fid(name(), scalar2d_layout_mid, m/s, grid_name); + FieldIdentifier fid(m_name, scalar2d_layout_mid, m/s, grid_name); m_diagnostic_output = Field(fid); m_diagnostic_output.get_header().get_alloc_properties().request_allocation(); m_diagnostic_output.allocate_view(); diff --git a/components/eamxx/src/diagnostics/precip_surf_mass_flux.hpp b/components/eamxx/src/diagnostics/precip_surf_mass_flux.hpp index 68dd1251646e..6ff3458398da 100644 --- a/components/eamxx/src/diagnostics/precip_surf_mass_flux.hpp +++ b/components/eamxx/src/diagnostics/precip_surf_mass_flux.hpp @@ -17,7 +17,7 @@ class PrecipSurfMassFlux : public AtmosphereDiagnostic PrecipSurfMassFlux (const ekat::Comm& comm, const ekat::ParameterList& params); // The name of the diagnostic - std::string name () const { return m_name; } + std::string name () const { return "PrecipSurfMassFlux"; } // Set the grid void set_grids (const std::shared_ptr grids_manager); diff --git a/components/eamxx/src/diagnostics/register_diagnostics.hpp b/components/eamxx/src/diagnostics/register_diagnostics.hpp index 45a04eebeb1f..67119416705d 100644 --- a/components/eamxx/src/diagnostics/register_diagnostics.hpp +++ b/components/eamxx/src/diagnostics/register_diagnostics.hpp @@ -24,6 +24,7 @@ #include "diagnostics/number_path.hpp" #include "diagnostics/aerocom_cld.hpp" #include "diagnostics/atm_backtend.hpp" +#include "diagnostics/horiz_avg.hpp" namespace scream { @@ -36,13 +37,6 @@ inline void register_diagnostics () { diag_factory.register_product("AtmosphereDensity",&create_atmosphere_diagnostic); diag_factory.register_product("Exner",&create_atmosphere_diagnostic); diag_factory.register_product("VirtualTemperature",&create_atmosphere_diagnostic); - diag_factory.register_product("z_int",&create_atmosphere_diagnostic); - diag_factory.register_product("z_mid",&create_atmosphere_diagnostic); - diag_factory.register_product("geopotential_int",&create_atmosphere_diagnostic); - diag_factory.register_product("geopotential_mid",&create_atmosphere_diagnostic); - diag_factory.register_product("height_int",&create_atmosphere_diagnostic); - diag_factory.register_product("height_mid",&create_atmosphere_diagnostic); - diag_factory.register_product("dz",&create_atmosphere_diagnostic); diag_factory.register_product("DryStaticEnergy",&create_atmosphere_diagnostic); diag_factory.register_product("SeaLevelPressure",&create_atmosphere_diagnostic); diag_factory.register_product("WaterPath",&create_atmosphere_diagnostic); @@ -50,6 +44,7 @@ inline void register_diagnostics () { diag_factory.register_product("LongwaveCloudForcing",&create_atmosphere_diagnostic); diag_factory.register_product("RelativeHumidity",&create_atmosphere_diagnostic); diag_factory.register_product("VaporFlux",&create_atmosphere_diagnostic); + diag_factory.register_product("VerticalLayer",&create_atmosphere_diagnostic); diag_factory.register_product("precip_surf_mass_flux",&create_atmosphere_diagnostic); diag_factory.register_product("surface_upward_latent_heat_flux",&create_atmosphere_diagnostic); diag_factory.register_product("wind_speed",&create_atmosphere_diagnostic); @@ -57,7 +52,9 @@ inline void register_diagnostics () { diag_factory.register_product("NumberPath",&create_atmosphere_diagnostic); diag_factory.register_product("AeroComCld",&create_atmosphere_diagnostic); diag_factory.register_product("AtmBackTendDiag",&create_atmosphere_diagnostic); + diag_factory.register_product("HorizAvgDiag",&create_atmosphere_diagnostic); } } // namespace scream + #endif // SCREAM_REGISTER_DIAGNOSTICS_HPP diff --git a/components/eamxx/src/diagnostics/tests/CMakeLists.txt b/components/eamxx/src/diagnostics/tests/CMakeLists.txt index a684aa248fc0..736253f9bee7 100644 --- a/components/eamxx/src/diagnostics/tests/CMakeLists.txt +++ b/components/eamxx/src/diagnostics/tests/CMakeLists.txt @@ -1,4 +1,4 @@ -# NOTE: tests inside this if statement won't be built in a baselines-only build +include(ScreamUtils) function (createDiagTest test_name test_srcs) CreateUnitTest(${test_name} "${test_srcs}" @@ -6,72 +6,71 @@ function (createDiagTest test_name test_srcs) LABELS diagnostics) endfunction () -if (NOT SCREAM_ONLY_GENERATE_BASELINES) - include(ScreamUtils) +# Test extracting a single level of a field +CreateDiagTest(field_at_level "field_at_level_tests.cpp") - # Test extracting a single level of a field - CreateDiagTest(field_at_level "field_at_level_tests.cpp") +# Test interpolating a field onto a single pressure level +CreateDiagTest(field_at_pressure_level "field_at_pressure_level_tests.cpp") - # Test interpolating a field onto a single pressure level - CreateDiagTest(field_at_pressure_level "field_at_pressure_level_tests.cpp") - # Test interpolating a field at a specific height - CreateDiagTest(field_at_height "field_at_height_tests.cpp") +# Test interpolating a field at a specific height +CreateDiagTest(field_at_height "field_at_height_tests.cpp") - # Test potential temperature diagnostic - CreateDiagTest(potential_temperature "potential_temperature_test.cpp") +# Test potential temperature diagnostic +CreateDiagTest(potential_temperature "potential_temperature_test.cpp") - # Test exner diagnostic - CreateDiagTest(exner_function "exner_test.cpp") +# Test exner diagnostic +CreateDiagTest(exner_function "exner_test.cpp") - # Test virtual temperature - CreateDiagTest(virtual_temperature "virtual_temperature_test.cpp") +# Test virtual temperature +CreateDiagTest(virtual_temperature "virtual_temperature_test.cpp") - # Test atmosphere density - CreateDiagTest(atmosphere_density "atm_density_test.cpp") +# Test atmosphere density +CreateDiagTest(atmosphere_density "atm_density_test.cpp") - # Test vertical layer (dz, z_int, z_mid) - CreateDiagTest(vertical_layer "vertical_layer_tests.cpp") +# Test vertical layer (dz, z_int, z_mid) +CreateDiagTest(vertical_layer "vertical_layer_tests.cpp") - # Test dry static energy - CreateDiagTest(dry_static_energy "dry_static_energy_test.cpp") +# Test dry static energy +CreateDiagTest(dry_static_energy "dry_static_energy_test.cpp") - # Test sea level pressure - CreateDiagTest(sea_level_pressure "sea_level_pressure_test.cpp") +# Test sea level pressure +CreateDiagTest(sea_level_pressure "sea_level_pressure_test.cpp") - # Test total water path - CreateDiagTest(water_path "water_path_tests.cpp") +# Test total water path +CreateDiagTest(water_path "water_path_tests.cpp") - # Test shortwave cloud forcing - CreateDiagTest(shortwave_cloud_forcing "shortwave_cloud_forcing_tests.cpp") +# Test shortwave cloud forcing +CreateDiagTest(shortwave_cloud_forcing "shortwave_cloud_forcing_tests.cpp") - # Test longwave cloud forcing - CreateDiagTest(longwave_cloud_forcing "longwave_cloud_forcing_tests.cpp") +# Test longwave cloud forcing +CreateDiagTest(longwave_cloud_forcing "longwave_cloud_forcing_tests.cpp") - # Test Relative Humidity - CreateDiagTest(relative_humidity "relative_humidity_tests.cpp") +# Test Relative Humidity +CreateDiagTest(relative_humidity "relative_humidity_tests.cpp") - # Test Vapor Flux - CreateDiagTest(vapor_flux "vapor_flux_tests.cpp") +# Test Vapor Flux +CreateDiagTest(vapor_flux "vapor_flux_tests.cpp") - # Test precipitation mass surface flux - CreateDiagTest(precip_surf_mass_flux "precip_surf_mass_flux_tests.cpp") +# Test precipitation mass surface flux +CreateDiagTest(precip_surf_mass_flux "precip_surf_mass_flux_tests.cpp") - # Test surface latent heat flux - CreateDiagTest(surface_upward_latent_heat_flux "surf_upward_latent_heat_flux_tests.cpp") +# Test surface latent heat flux +CreateDiagTest(surface_upward_latent_heat_flux "surf_upward_latent_heat_flux_tests.cpp") - # Test wind speed diagnostic - CreateDiagTest(wind_speed "wind_speed_tests.cpp") +# Test wind speed diagnostic +CreateDiagTest(wind_speed "wind_speed_tests.cpp") - # Test AODVIS - CreateDiagTest(aodvis "aodvis_test.cpp") +# Test AODVIS +CreateDiagTest(aodvis "aodvis_test.cpp") - # Test "number" paths - CreateDiagTest(number_paths "number_paths_tests.cpp") +# Test "number" paths +CreateDiagTest(number_paths "number_paths_tests.cpp") - # Test AEROCOM_CLD - CreateDiagTest(aerocom_cld "aerocom_cld_test.cpp") +# Test AEROCOM_CLD +CreateDiagTest(aerocom_cld "aerocom_cld_test.cpp") - # Test atm_tend - CreateDiagTest(atm_backtend "atm_backtend_test.cpp") +# Test atm_tend +CreateDiagTest(atm_backtend "atm_backtend_test.cpp") -endif() +# Test horizontal averaging +CreateDiagTest(horiz_avg "horiz_avg_test.cpp") diff --git a/components/eamxx/src/diagnostics/tests/field_at_height_tests.cpp b/components/eamxx/src/diagnostics/tests/field_at_height_tests.cpp index e26d34dd1a55..6e071e4b2013 100644 --- a/components/eamxx/src/diagnostics/tests/field_at_height_tests.cpp +++ b/components/eamxx/src/diagnostics/tests/field_at_height_tests.cpp @@ -104,12 +104,13 @@ TEST_CASE("field_at_height") // Lambda to create and run a diag, and return output auto run_diag = [&](const Field& f, const Field& z, - const std::string& loc, const std::string& surf_ref) { + const double h, const std::string& surf_ref) { util::TimeStamp t0 ({2022,1,1},{0,0,0}); auto& factory = AtmosphereDiagnosticFactory::instance(); ekat::ParameterList pl; pl.set("surface_reference",surf_ref); - pl.set("vertical_location",loc); + pl.set("height_value",std::to_string(h)); + pl.set("height_units",std::string("m")); pl.set("field_name",f.name()); pl.set("grid_name",grid->name()); auto diag = factory.create("FieldAtheight",comm,pl); @@ -173,13 +174,12 @@ TEST_CASE("field_at_height") // Make sure that an unsupported reference height throws an error. print(" -> Testing throws error with unsupported reference height...\n"); { - REQUIRE_THROWS(run_diag (s_mid,h_mid,"1m","foobar")); + REQUIRE_THROWS(run_diag (s_mid,h_mid,1.0,"foobar")); } print(" -> Testing throws error with unsupported reference height... OK\n"); // Run many times int z_tgt; - std::string loc; for (std::string surf_ref : {"sealevel","surface"}) { printf(" -> Testing for a reference height above %s...\n",surf_ref.c_str()); const auto mid_src = surf_ref == "sealevel" ? z_mid : h_mid; @@ -197,32 +197,31 @@ TEST_CASE("field_at_height") // Set target z-slice for testing to a random value. z_tgt = pdf_levs(engine)+max_surf_4test; - loc = std::to_string(z_tgt) + "m"; - printf(" -> test at height of %s.............\n",loc.c_str()); + printf(" -> test at height of %dm............\n",z_tgt); { print(" -> scalar midpoint field...............\n"); - auto d = run_diag(s_mid,mid_src,loc,surf_ref); + auto d = run_diag(s_mid,mid_src,z_tgt,surf_ref); f_z_tgt(inter,slope,z_tgt,mid_src,s_tgt); REQUIRE (views_are_approx_equal(d,s_tgt,tol)); print(" -> scalar midpoint field............... OK!\n"); } { print(" -> scalar interface field...............\n"); - auto d = run_diag (s_int,int_src,loc,surf_ref); + auto d = run_diag (s_int,int_src,z_tgt,surf_ref); f_z_tgt(inter,slope,z_tgt,int_src,s_tgt); REQUIRE (views_are_approx_equal(d,s_tgt,tol)); print(" -> scalar interface field............... OK!\n"); } { print(" -> vector midpoint field...............\n"); - auto d = run_diag (v_mid,mid_src,loc,surf_ref); + auto d = run_diag (v_mid,mid_src,z_tgt,surf_ref); f_z_tgt(inter,slope,z_tgt,mid_src,v_tgt); REQUIRE (views_are_approx_equal(d,v_tgt,tol)); print(" -> vector midpoint field............... OK!\n"); } { print(" -> vector interface field...............\n"); - auto d = run_diag (v_int,int_src,loc,surf_ref); + auto d = run_diag (v_int,int_src,z_tgt,surf_ref); f_z_tgt(inter,slope,z_tgt,int_src,v_tgt); REQUIRE (views_are_approx_equal(d,v_tgt,tol)); print(" -> vector interface field............... OK!\n"); @@ -230,8 +229,7 @@ TEST_CASE("field_at_height") { print(" -> Forced fail, give incorrect location...............\n"); const int z_tgt_adj = (z_tgt+max_surf_4test)/2; - std::string loc_err = std::to_string(z_tgt_adj) + "m"; - auto d = run_diag(s_int,int_src,loc_err,surf_ref); + auto d = run_diag(s_int,int_src,z_tgt_adj,surf_ref); f_z_tgt(inter,slope,z_tgt,int_src,s_tgt); REQUIRE (!views_are_approx_equal(d,s_tgt,tol,false)); print(" -> Forced fail, give incorrect location............... OK!\n"); @@ -243,15 +241,13 @@ TEST_CASE("field_at_height") auto inter = pdf_y0(engine); f_z_src(inter, slope, int_src, s_int); z_tgt = 2*z_top; - std::string loc = std::to_string(z_tgt) + "m"; - auto dtop = run_diag(s_int,int_src,loc,surf_ref); + auto dtop = run_diag(s_int,int_src,z_tgt,surf_ref); f_z_tgt(inter,slope,z_tgt,int_src,s_tgt); REQUIRE (views_are_approx_equal(dtop,s_tgt,tol)); print(" -> Forced extrapolation at top............... OK!\n"); print(" -> Forced extrapolation at bot...............\n"); z_tgt = 0; - loc = std::to_string(z_tgt) + "m"; - auto dbot = run_diag(s_int,int_src,loc,surf_ref); + auto dbot = run_diag(s_int,int_src,z_tgt,surf_ref); f_z_tgt(inter,slope,z_tgt,int_src,s_tgt); REQUIRE (views_are_approx_equal(dbot,s_tgt,tol)); print(" -> Forced extrapolation at bot............... OK!\n"); diff --git a/components/eamxx/src/diagnostics/tests/field_at_pressure_level_tests.cpp b/components/eamxx/src/diagnostics/tests/field_at_pressure_level_tests.cpp index 4e0deab1dfca..ba733980cade 100644 --- a/components/eamxx/src/diagnostics/tests/field_at_pressure_level_tests.cpp +++ b/components/eamxx/src/diagnostics/tests/field_at_pressure_level_tests.cpp @@ -224,7 +224,8 @@ get_test_diag(const ekat::Comm& comm, std::shared_ptr fm, st ekat::ParameterList params; params.set("field_name",field.name()); params.set("grid_name",fm->get_grid()->name()); - params.set("vertical_location",std::to_string(plevel) + "Pa"); + params.set("pressure_value",std::to_string(plevel)); + params.set("pressure_units",std::string("Pa")); auto diag = std::make_shared(comm,params); diag->set_grids(gm); for (const auto& req : diag->get_required_field_requests()) { diff --git a/components/eamxx/src/diagnostics/tests/horiz_avg_test.cpp b/components/eamxx/src/diagnostics/tests/horiz_avg_test.cpp new file mode 100644 index 000000000000..9b9ec2022fa0 --- /dev/null +++ b/components/eamxx/src/diagnostics/tests/horiz_avg_test.cpp @@ -0,0 +1,166 @@ +#include "catch2/catch.hpp" +#include "diagnostics/register_diagnostics.hpp" +#include "share/field/field_utils.hpp" +#include "share/grid/mesh_free_grids_manager.hpp" +#include "share/util/scream_setup_random_test.hpp" +#include "share/util/scream_universal_constants.hpp" + +namespace scream { + +std::shared_ptr create_gm(const ekat::Comm &comm, const int ncols, + const int nlevs) { + const int num_global_cols = ncols * comm.size(); + + using vos_t = std::vector; + ekat::ParameterList gm_params; + gm_params.set("grids_names", vos_t{"Point Grid"}); + auto &pl = gm_params.sublist("Point Grid"); + pl.set("type", "point_grid"); + pl.set("aliases", vos_t{"Physics"}); + pl.set("number_of_global_columns", num_global_cols); + pl.set("number_of_vertical_levels", nlevs); + + auto gm = create_mesh_free_grids_manager(comm, gm_params); + gm->build_grids(); + + return gm; +} + +TEST_CASE("horiz_avg") { + using namespace ShortFieldTagsNames; + using namespace ekat::units; + + // A numerical tolerance + auto tol = std::numeric_limits::epsilon() * 100; + + // A world comm + ekat::Comm comm(MPI_COMM_WORLD); + + // A time stamp + util::TimeStamp t0({2024, 1, 1}, {0, 0, 0}); + + // Create a grids manager - single column for these tests + constexpr int nlevs = 3; + constexpr int dim3 = 4; + const int ngcols = 6 * comm.size(); + + auto gm = create_gm(comm, ngcols, nlevs); + auto grid = gm->get_grid("Physics"); + + // Input (randomized) qc + FieldLayout scalar1d_layout{{COL}, {ngcols}}; + FieldLayout scalar2d_layout{{COL, LEV}, {ngcols, nlevs}}; + FieldLayout scalar3d_layout{{COL, CMP, LEV}, {ngcols, dim3, nlevs}}; + + FieldIdentifier qc1_fid("qc", scalar1d_layout, kg / kg, grid->name()); + FieldIdentifier qc2_fid("qc", scalar2d_layout, kg / kg, grid->name()); + FieldIdentifier qc3_fid("qc", scalar3d_layout, kg / kg, grid->name()); + + Field qc1(qc1_fid); + Field qc2(qc2_fid); + Field qc3(qc3_fid); + + qc1.allocate_view(); + qc2.allocate_view(); + qc3.allocate_view(); + + // Construct random number generator stuff + using RPDF = std::uniform_real_distribution; + RPDF pdf(sp(0.0), sp(200.0)); + auto engine = scream::setup_random_test(); + + // Construct the Diagnostics + std::map> diags; + auto &diag_factory = AtmosphereDiagnosticFactory::instance(); + register_diagnostics(); + + ekat::ParameterList params; + REQUIRE_THROWS(diag_factory.create("HorizAvgDiag", comm, + params)); // No 'field_name' parameter + + // Set time for qc and randomize its values + qc1.get_header().get_tracking().update_time_stamp(t0); + qc2.get_header().get_tracking().update_time_stamp(t0); + qc3.get_header().get_tracking().update_time_stamp(t0); + randomize(qc1, engine, pdf); + randomize(qc2, engine, pdf); + randomize(qc3, engine, pdf); + + // Create and set up the diagnostic + params.set("grid_name", grid->name()); + params.set("field_name", "qc"); + auto diag1 = diag_factory.create("HorizAvgDiag", comm, params); + auto diag2 = diag_factory.create("HorizAvgDiag", comm, params); + auto diag3 = diag_factory.create("HorizAvgDiag", comm, params); + diag1->set_grids(gm); + diag2->set_grids(gm); + diag3->set_grids(gm); + + // Clone the area field + auto area = grid->get_geometry_data("area").clone(); + + // Test the horiz contraction of qc1 + // Get the diagnostic field + diag1->set_required_field(qc1); + diag1->initialize(t0, RunType::Initial); + diag1->compute_diagnostic(); + auto diag1_f = diag1->get_diagnostic(); + + // Manual calculation + FieldIdentifier diag0_fid("qc_horiz_avg_manual", + scalar1d_layout.clone().strip_dim(COL), kg / kg, + grid->name()); + Field diag0(diag0_fid); + diag0.allocate_view(); + + // calculate total area + Real atot = field_sum(area, &comm); + // scale the area field + area.scale(1 / atot); + + // calculate weighted avg + horiz_contraction(diag0, qc1, area, &comm); + // Compare + REQUIRE(views_are_equal(diag1_f, diag0)); + + // Try other known cases + // Set qc1_v to 1.0 to get weighted average of 1.0 + Real wavg = 1; + qc1.deep_copy(wavg); + diag1->compute_diagnostic(); + auto diag1_v2_host = diag1_f.get_view(); + REQUIRE_THAT(diag1_v2_host(), + Catch::Matchers::WithinRel( + wavg, tol)); // Catch2's floating point comparison + + // other diags + // Set qc2_v to 5.0 to get weighted average of 5.0 + wavg = sp(5.0); + qc2.deep_copy(wavg); + diag2->set_required_field(qc2); + diag2->initialize(t0, RunType::Initial); + diag2->compute_diagnostic(); + auto diag2_f = diag2->get_diagnostic(); + + auto diag2_v_host = diag2_f.get_view(); + + for(int i = 0; i < nlevs; ++i) { + REQUIRE_THAT(diag2_v_host(i), Catch::Matchers::WithinRel(wavg, tol)); + } + + // Try a random case with qc3 + auto qc3_v = qc3.get_view(); + FieldIdentifier diag3_manual_fid("qc_horiz_avg_manual", + scalar3d_layout.clone().strip_dim(COL), + kg / kg, grid->name()); + Field diag3_manual(diag3_manual_fid); + diag3_manual.allocate_view(); + horiz_contraction(diag3_manual, qc3, area, &comm); + diag3->set_required_field(qc3); + diag3->initialize(t0, RunType::Initial); + diag3->compute_diagnostic(); + auto diag3_f = diag3->get_diagnostic(); + REQUIRE(views_are_equal(diag3_f, diag3_manual)); +} + +} // namespace scream diff --git a/components/eamxx/src/diagnostics/tests/vertical_layer_tests.cpp b/components/eamxx/src/diagnostics/tests/vertical_layer_tests.cpp index fe75114611d5..631cb5acd8a8 100644 --- a/components/eamxx/src/diagnostics/tests/vertical_layer_tests.cpp +++ b/components/eamxx/src/diagnostics/tests/vertical_layer_tests.cpp @@ -50,14 +50,10 @@ void run (const std::string& diag_name, const std::string& location) // Construct the Diagnostic ekat::ParameterList params; - std::string name = diag_name; - if (location=="midpoints") { - name += "_mid"; - } else if (location=="interfaces") { - name += "_int"; - } - params.set("diag_name", name); - auto diag = diag_factory.create(name,comm,params); + + params.set("diag_name", diag_name); + params.set("vert_location",location); + auto diag = diag_factory.create("VerticalLayer",comm,params); diag->set_grids(gm); const bool needs_phis = diag_name=="z" or diag_name=="geopotential"; @@ -180,7 +176,7 @@ TEST_CASE("vertical_layer_test", "vertical_layer_test]"){ std::string msg = " -> Testing diag=dz "; std::string dots (50-msg.size(),'.'); root_print (msg + dots + "\n"); - run("dz", "UNUSED"); + run("dz", "midpoints"); root_print (msg + dots + " PASS!\n"); }; diff --git a/components/eamxx/src/diagnostics/tests/wind_speed_tests.cpp b/components/eamxx/src/diagnostics/tests/wind_speed_tests.cpp index 5e6be61ba4f0..1298536c4232 100644 --- a/components/eamxx/src/diagnostics/tests/wind_speed_tests.cpp +++ b/components/eamxx/src/diagnostics/tests/wind_speed_tests.cpp @@ -62,6 +62,11 @@ TEST_CASE("wind_speed") register_diagnostics(); constexpr int ntests = 5; +#ifdef NDEBUG + constexpr int ulp_tol = 1; +#else + constexpr int ulp_tol = 0; +#endif for (int itest=0; itest grids_manager) { using namespace ekat::units; @@ -51,7 +46,8 @@ void VaporFluxDiagnostic::set_grids(const std::shared_ptr gr add_field("horiz_winds", vector3d, m/s, grid_name); // Construct and allocate the diagnostic field - FieldIdentifier fid (name(), scalar2d, kg/m/s, grid_name); + std::string dname = m_component==0 ? "ZonalVapFlux" : "MeridionalVapFlux"; + FieldIdentifier fid (dname, scalar2d, kg/m/s, grid_name); m_diagnostic_output = Field(fid); m_diagnostic_output.allocate_view(); } diff --git a/components/eamxx/src/diagnostics/vapor_flux.hpp b/components/eamxx/src/diagnostics/vapor_flux.hpp index 3d82fd882f8c..5bacd78a9b71 100644 --- a/components/eamxx/src/diagnostics/vapor_flux.hpp +++ b/components/eamxx/src/diagnostics/vapor_flux.hpp @@ -17,7 +17,7 @@ class VaporFluxDiagnostic : public AtmosphereDiagnostic VaporFluxDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); // The name of the diagnostic - std::string name () const; + std::string name () const override { return "VaporFlux"; } // Set the grid void set_grids (const std::shared_ptr grids_manager); diff --git a/components/eamxx/src/diagnostics/vertical_layer.cpp b/components/eamxx/src/diagnostics/vertical_layer.cpp index 32d870da03cc..f9ff526dfcd8 100644 --- a/components/eamxx/src/diagnostics/vertical_layer.cpp +++ b/components/eamxx/src/diagnostics/vertical_layer.cpp @@ -13,25 +13,27 @@ VerticalLayerDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& para : AtmosphereDiagnostic(comm,params) { m_diag_name = params.get("diag_name"); - std::vector supported = { - "z_int", - "z_mid", - "geopotential_int", - "geopotential_mid", - "height_int", - "height_mid", - "dz" - }; + std::vector supported = {"z","geopotential","height","dz"}; EKAT_REQUIRE_MSG(ekat::contains(supported,m_diag_name), "[VerticalLayerDiagnostic] Error! Invalid diag_name.\n" " - diag_name : " + m_diag_name + "\n" " - valid names: " + ekat::join(supported,", ") + "\n"); - m_is_interface_layout = m_diag_name.find("_int") != std::string::npos; + auto vert_pos = params.get("vert_location"); + EKAT_REQUIRE_MSG (vert_pos=="mid" || vert_pos=="int" || + vert_pos=="midpoints" || vert_pos=="interfaces", + "[VerticalLayerDiagnostic] Error! Invalid 'vert_location'.\n" + " - input value: " + vert_pos + "\n" + " - valid names: mid, midpoints, int, interfaces\n"); + m_is_interface_layout = vert_pos=="int" || vert_pos=="interfaces"; + + m_geopotential = m_diag_name=="geopotential"; + m_from_sea_level = m_diag_name=="z" or m_geopotential; - m_geopotential = m_diag_name.substr(0,12)=="geopotential"; - m_from_sea_level = m_diag_name[0]=='z' or m_geopotential; + if (m_diag_name!="dz") { + m_diag_name += m_is_interface_layout ? "_int" : "_mid"; + } } // ======================================================================================== void VerticalLayerDiagnostic:: @@ -88,7 +90,7 @@ initialize_impl (const RunType /*run_type*/) const auto VLEV = m_is_interface_layout ? ILEV : LEV; const auto nlevs = m_is_interface_layout ? m_num_levs+1 : m_num_levs; FieldLayout diag_layout ({COL,VLEV},{m_num_cols,nlevs}); - FieldIdentifier fid (name(), diag_layout, m_geopotential ? m2/s2 : m, grid_name); + FieldIdentifier fid (m_diag_name, diag_layout, m_geopotential ? m2/s2 : m, grid_name); m_diagnostic_output = Field(fid); auto& diag_fap = m_diagnostic_output.get_header().get_alloc_properties(); diff --git a/components/eamxx/src/diagnostics/vertical_layer.hpp b/components/eamxx/src/diagnostics/vertical_layer.hpp index 805fd70028f6..a440bd6a8ee9 100644 --- a/components/eamxx/src/diagnostics/vertical_layer.hpp +++ b/components/eamxx/src/diagnostics/vertical_layer.hpp @@ -24,7 +24,7 @@ class VerticalLayerDiagnostic : public AtmosphereDiagnostic VerticalLayerDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); // The name of the diagnostic. - std::string name () const { return m_diag_name; } + std::string name () const { return "VerticalLayer"; } // Set the grid void set_grids (const std::shared_ptr grids_manager); diff --git a/components/eamxx/src/diagnostics/water_path.cpp b/components/eamxx/src/diagnostics/water_path.cpp index 15fa5cdef38c..bca771e6dab6 100644 --- a/components/eamxx/src/diagnostics/water_path.cpp +++ b/components/eamxx/src/diagnostics/water_path.cpp @@ -32,11 +32,6 @@ WaterPathDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) } } -std::string WaterPathDiagnostic::name() const -{ - return m_kind + "WaterPath"; -} - void WaterPathDiagnostic:: set_grids(const std::shared_ptr grids_manager) { @@ -57,7 +52,7 @@ set_grids(const std::shared_ptr grids_manager) add_field(m_qname, scalar3d, kg/kg, grid_name); // Construct and allocate the diagnostic field - FieldIdentifier fid (name(), scalar2d, kg/m2, grid_name); + FieldIdentifier fid (m_kind + "WaterPath", scalar2d, kg/m2, grid_name); m_diagnostic_output = Field(fid); m_diagnostic_output.allocate_view(); } diff --git a/components/eamxx/src/diagnostics/water_path.hpp b/components/eamxx/src/diagnostics/water_path.hpp index 0b9515e0b421..722a51a31331 100644 --- a/components/eamxx/src/diagnostics/water_path.hpp +++ b/components/eamxx/src/diagnostics/water_path.hpp @@ -17,7 +17,7 @@ class WaterPathDiagnostic : public AtmosphereDiagnostic WaterPathDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); // The name of the diagnostic - std::string name () const; + std::string name () const override { return "WaterPath"; } // Set the grid void set_grids (const std::shared_ptr grids_manager); diff --git a/components/eamxx/src/dynamics/homme/CMakeLists.txt b/components/eamxx/src/dynamics/homme/CMakeLists.txt index c7dfef3f8df3..ffad3a4111f9 100644 --- a/components/eamxx/src/dynamics/homme/CMakeLists.txt +++ b/components/eamxx/src/dynamics/homme/CMakeLists.txt @@ -147,7 +147,6 @@ macro (CreateDynamicsLib HOMME_TARGET NP PLEV QSIZE) ${SCREAM_DYNAMICS_SRC_DIR}/eamxx_homme_process_interface.cpp ${SCREAM_DYNAMICS_SRC_DIR}/eamxx_homme_fv_phys.cpp ${SCREAM_DYNAMICS_SRC_DIR}/eamxx_homme_rayleigh_friction.cpp - ${SCREAM_DYNAMICS_SRC_DIR}/eamxx_homme_iop.cpp ${SCREAM_DYNAMICS_SRC_DIR}/physics_dynamics_remapper.cpp ${SCREAM_DYNAMICS_SRC_DIR}/homme_grids_manager.cpp ${SCREAM_DYNAMICS_SRC_DIR}/interface/homme_context_mod.F90 diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_fv_phys.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_fv_phys.cpp index a121600536b8..aae0e549cd76 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_fv_phys.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_fv_phys.cpp @@ -233,9 +233,12 @@ void HommeDynamics::remap_fv_phys_to_dyn () const { // See the [rrtmgp active gases] note in share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.hpp void HommeDynamics -::fv_phys_rrtmgp_active_gases_init (const std::shared_ptr& gm) { +::fv_phys_rrtmgp_active_gases_init (const std::shared_ptr& gm) +{ + // NOTE: we would like to avoid this if it's a restart run, but at this point of the + // init sequence we still don't know the run type. So we must add the trace gases + // fields, and we will deal with them later auto& trace_gases_workaround = TraceGasesWorkaround::singleton(); - if (trace_gases_workaround.is_restart()) return; // always false b/c it hasn't been set yet using namespace ekat::units; using namespace ShortFieldTagsNames; const auto& rgn = m_cgll_grid->name(); @@ -254,7 +257,7 @@ ::fv_phys_rrtmgp_active_gases_init (const std::shared_ptr& g } // See the [rrtmgp active gases] note in share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.hpp -void HommeDynamics::fv_phys_rrtmgp_active_gases_remap () { +void HommeDynamics::fv_phys_rrtmgp_active_gases_remap (const RunType run_type) { // Note re: restart: Ideally, we'd know if we're restarting before having to // call add_field above. However, we only find out after. Because the pg2 // field was declared Updated, it will read the restart data. But we don't @@ -262,7 +265,7 @@ void HommeDynamics::fv_phys_rrtmgp_active_gases_remap () { // cleanup part at the end. auto& trace_gases_workaround = TraceGasesWorkaround::singleton(); const auto& rgn = m_cgll_grid->name(); - if (not trace_gases_workaround.is_restart()) { + if (run_type==RunType::Initial) { using namespace ShortFieldTagsNames; const auto& dgn = m_dyn_grid ->name(); const auto& pgn = m_phys_grid->name(); diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp deleted file mode 100644 index cf70dd49f564..000000000000 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ /dev/null @@ -1,613 +0,0 @@ -#include "eamxx_homme_process_interface.hpp" - -// EAMxx includes -#include "dynamics/homme/homme_dimensions.hpp" -#include "dynamics/homme/homme_dynamics_helpers.hpp" -#include "physics/share/physics_constants.hpp" -#include "share/iop/intensive_observation_period.hpp" -#include "share/util/scream_column_ops.hpp" - -// Homme includes -#include "Context.hpp" -#include "ColumnOps.hpp" -#include "HommexxEnums.hpp" -#include "HybridVCoord.hpp" -#include "SimulationParams.hpp" -#include "Types.hpp" - -// SCREAM includes -#include "share/util/scream_common_physics_functions.hpp" - -// EKAT includes -#include "ekat/ekat_workspace.hpp" -#include "ekat/kokkos/ekat_kokkos_types.hpp" - -namespace scream { - -// Compute effects of large scale subsidence on T, q, u, and v. -KOKKOS_FUNCTION -void HommeDynamics:: -advance_iop_subsidence(const KT::MemberType& team, - const int nlevs, - const Real dt, - const Real ps, - const view_1d& pmid, - const view_1d& pint, - const view_1d& pdel, - const view_1d& omega, - const Workspace& workspace, - const view_1d& u, - const view_1d& v, - const view_1d& T, - const view_2d& Q) -{ - using ColOps = ColumnOps; - using C = physics::Constants; - constexpr Real Rair = C::Rair; - constexpr Real Cpair = C::Cpair; - - const auto n_q_tracers = Q.extent_int(0); - const auto nlev_packs = ekat::npack(nlevs); - - // Get some temporary views from WS - uview_1d omega_int, delta_u, delta_v, delta_T, tmp; - workspace.take_many_contiguous_unsafe<4>({"omega_int", "delta_u", "delta_v", "delta_T"}, - {&omega_int, &delta_u, &delta_v, &delta_T}); - const auto delta_Q_slot = workspace.take_macro_block("delta_Q", n_q_tracers); - uview_2d delta_Q(delta_Q_slot.data(), n_q_tracers, nlev_packs); - - auto s_pmid = ekat::scalarize(pmid); - auto s_omega = ekat::scalarize(omega); - auto s_delta_u = ekat::scalarize(delta_u); - auto s_delta_v = ekat::scalarize(delta_v); - auto s_delta_T = ekat::scalarize(delta_T); - auto s_delta_Q = ekat::scalarize(delta_Q); - auto s_omega_int = ekat::scalarize(omega_int); - - // Compute omega on the interface grid by using a weighted average in pressure - const int pack_begin = 1/Pack::n, pack_end = (nlevs-1)/Pack::n; - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, pack_begin, pack_end+1), [&] (const int k){ - auto range_pack = ekat::range(k*Pack::n); - range_pack.set(range_pack<1, 1); - Pack pmid_k, pmid_km1, omega_k, omega_km1; - ekat::index_and_shift<-1>(s_pmid, range_pack, pmid_k, pmid_km1); - ekat::index_and_shift<-1>(s_omega, range_pack, omega_k, omega_km1); - - const auto weight = (pint(k) - pmid_km1)/(pmid_k - pmid_km1); - omega_int(k).set(range_pack>=1 and range_pack<=nlevs-1, - weight*omega_k + (1-weight)*omega_km1); - }); - omega_int(0)[0] = 0; - omega_int(nlevs/Pack::n)[nlevs%Pack::n] = 0; - - // Compute delta views for u, v, T, and Q (e.g., u(k+1) - u(k), k=0,...,nlevs-2) - ColOps::compute_midpoint_delta(team, nlevs-1, u, delta_u); - ColOps::compute_midpoint_delta(team, nlevs-1, v, delta_v); - ColOps::compute_midpoint_delta(team, nlevs-1, T, delta_T); - for (int iq=0; iq(k*Pack::n); - const auto at_top = range_pack==0; - const auto not_at_top = not at_top; - const auto at_bot = range_pack==nlevs-1; - const auto not_at_bot = not at_bot; - const bool any_at_top = at_top.any(); - const bool any_at_bot = at_bot.any(); - - // Get delta(k-1) packs. The range pack should not - // contain index 0 (so that we don't attempt to access - // k=-1 index) or index > nlevs-2 (since delta_* views - // are size nlevs-1). - auto range_pack_for_m1_shift = range_pack; - range_pack_for_m1_shift.set(range_pack<1, 1); - range_pack_for_m1_shift.set(range_pack>nlevs-2, nlevs-2); - Pack delta_u_k, delta_u_km1, - delta_v_k, delta_v_km1, - delta_T_k, delta_T_km1; - ekat::index_and_shift<-1>(s_delta_u, range_pack_for_m1_shift, delta_u_k, delta_u_km1); - ekat::index_and_shift<-1>(s_delta_v, range_pack_for_m1_shift, delta_v_k, delta_v_km1); - ekat::index_and_shift<-1>(s_delta_T, range_pack_for_m1_shift, delta_T_k, delta_T_km1); - - // At the top and bottom of the model, set the end points for - // delta_*_k and delta_*_km1 to be the first and last entries - // of delta_*, respectively. - if (any_at_top) { - delta_u_k.set(at_top, s_delta_u(0)); - delta_v_k.set(at_top, s_delta_v(0)); - delta_T_k.set(at_top, s_delta_T(0)); - } - if (any_at_bot) { - delta_u_km1.set(at_bot, s_delta_u(nlevs-2)); - delta_v_km1.set(at_bot, s_delta_v(nlevs-2)); - delta_T_km1.set(at_bot, s_delta_T(nlevs-2)); - } - - // Get omega_int(k+1) pack. The range pack should not - // contain index > nlevs-1 (since omega_int is size nlevs+1). - auto range_pack_for_p1_shift = range_pack; - range_pack_for_p1_shift.set(range_pack>nlevs-1, nlevs-1); - Pack omega_int_k, omega_int_kp1; - ekat::index_and_shift<1>(s_omega_int, range_pack, omega_int_k, omega_int_kp1); - - const auto fac = (dt/2)/pdel(k); - - // Update u - u(k).update(not_at_bot, fac*omega_int_kp1*delta_u_k, -1, 1); - u(k).update(not_at_top, fac*omega_int_k*delta_u_km1, -1, 1); - - // Update v - v(k).update(not_at_bot, fac*omega_int_kp1*delta_v_k, -1, 1); - v(k).update(not_at_top, fac*omega_int_k*delta_v_km1, -1, 1); - - // Before updating T, first scale using thermal - // expansion term due to LS vertical advection - T(k) *= 1 + (dt*Rair/Cpair)*omega(k)/pmid(k); - - // Update T - T(k).update(not_at_bot, fac*omega_int_kp1*delta_T_k, -1, 1); - T(k).update(not_at_top, fac*omega_int_k*delta_T_km1, -1, 1); - - // Update Q - Pack delta_tracer_k, delta_tracer_km1; - for (int iq=0; iq(s_delta_tracer, range_pack_for_m1_shift, delta_tracer_k, delta_tracer_km1); - if (any_at_top) delta_tracer_k.set(at_top, s_delta_tracer(0)); - if (any_at_bot) delta_tracer_km1.set(at_bot, s_delta_tracer(nlevs-2)); - - Q(iq, k).update(not_at_bot, fac*omega_int_kp1*delta_tracer_k, -1, 1); - Q(iq, k).update(not_at_top, fac*omega_int_k*delta_tracer_km1, -1, 1); - } - }); - - // Release WS views - workspace.release_macro_block(delta_Q_slot, n_q_tracers); - workspace.release_many_contiguous<4>({&omega_int, &delta_u, &delta_v, &delta_T}); -} - -// Apply large scale forcing for temperature and water vapor as provided by the IOP file -KOKKOS_FUNCTION -void HommeDynamics:: -advance_iop_forcing(const KT::MemberType& team, - const int nlevs, - const Real dt, - const view_1d& divT, - const view_1d& divq, - const view_1d& T, - const view_1d& qv) -{ - const auto nlev_packs = ekat::npack(nlevs); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_packs), [&] (const int k) { - T(k).update(divT(k), dt, 1.0); - qv(k).update(divq(k), dt, 1.0); - }); -} - -// Provide coriolis forcing to u and v winds, using large scale winds specified in IOP forcing file. -KOKKOS_FUNCTION -void HommeDynamics:: -iop_apply_coriolis(const KT::MemberType& team, - const int nlevs, - const Real dt, - const Real lat, - const view_1d& u_ls, - const view_1d& v_ls, - const view_1d& u, - const view_1d& v) -{ - using C = physics::Constants; - constexpr Real pi = C::Pi; - constexpr Real earth_rotation = C::omega; - - // Compute coriolis force - const auto fcor = 2*earth_rotation*std::sin(lat*pi/180); - - const auto nlev_packs = ekat::npack(nlevs); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_packs), [&] (const int k) { - const auto u_cor = v(k) - v_ls(k); - const auto v_cor = u(k) - u_ls(k); - u(k).update(u_cor, dt*fcor, 1.0); - v(k).update(v_cor, -dt*fcor, 1.0); - }); -} - -void HommeDynamics:: -apply_iop_forcing(const Real dt) -{ - using ESU = ekat::ExeSpaceUtils; - using PF = PhysicsFunctions; - using ColOps = ColumnOps; - using C = physics::Constants; - constexpr Real Rair = C::Rair; - - // Homme objects - const auto& c = Homme::Context::singleton(); - const auto& hvcoord = c.get(); - const auto& params = c.get(); - - // Dimensions - constexpr int NGP = HOMMEXX_NP; - constexpr int NLEV = HOMMEXX_NUM_LEV; - constexpr int NLEVI = HOMMEXX_NUM_LEV_P; - const auto nelem = m_dyn_grid->get_num_local_dofs()/(NGP*NGP); - const auto total_levels = m_dyn_grid->get_num_vertical_levels(); - const auto qsize = params.qsize; - const auto theta_hydrostatic_mode = params.theta_hydrostatic_mode; - - // Sanity checks since we will be switching between ekat::Pack - // and Homme::Scalar view types - EKAT_ASSERT_MSG(NLEV == ekat::npack(total_levels), - "Error! Dimension for vectorized Homme levels does not match level dimension " - "of the packed views used here. Check that Pack typedef is using a pack size " - "consistent with Homme's vector size.\n"); - EKAT_ASSERT_MSG(NLEVI == ekat::npack(total_levels+1), - "Error! Dimension for vectorized Homme levels does not match level dimension " - "of the packed views used here. Check that Pack typedef is using a pack size " - "consistent with Homme's vector size.\n"); - - // Hybrid coord values - const auto ps0 = hvcoord.ps0; - const auto hyam = m_dyn_grid->get_geometry_data("hyam").get_view(); - const auto hybm = m_dyn_grid->get_geometry_data("hybm").get_view(); - const auto hyai = m_dyn_grid->get_geometry_data("hyai").get_view(); - const auto hybi = m_dyn_grid->get_geometry_data("hybi").get_view(); - - // Homme element states - auto ps_dyn = get_internal_field("ps_dyn").get_view(); - auto dp3d_dyn = get_internal_field("dp3d_dyn").get_view(); - auto vtheta_dp_dyn = get_internal_field("vtheta_dp_dyn").get_view(); - auto phi_int_dyn = get_internal_field("phi_int_dyn").get_view(); - auto v_dyn = get_internal_field("v_dyn").get_view(); - auto Q_dyn = m_helper_fields.at("Q_dyn").get_view(); - auto Qdp_dyn = get_internal_field("Qdp_dyn").get_view(); - - // Load data from IOP files, if necessary - m_iop->read_iop_file_data(timestamp()); - - // Define local IOP param values - const auto iop_dosubsidence = m_iop->get_params().get("iop_dosubsidence"); - const auto iop_coriolis = m_iop->get_params().get("iop_coriolis"); - const auto iop_nudge_tq = m_iop->get_params().get("iop_nudge_tq"); - const auto iop_nudge_uv = m_iop->get_params().get("iop_nudge_uv"); - const auto use_large_scale_wind = m_iop->get_params().get("use_large_scale_wind"); - const auto use_3d_forcing = m_iop->get_params().get("use_3d_forcing"); - const auto lat = m_iop->get_params().get("target_latitude"); - const auto iop_nudge_tscale = m_iop->get_params().get("iop_nudge_tscale"); - const auto iop_nudge_tq_low = m_iop->get_params().get("iop_nudge_tq_low"); - const auto iop_nudge_tq_high = m_iop->get_params().get("iop_nudge_tq_high"); - - // Define local IOP field views - const Real ps_iop = m_iop->get_iop_field("Ps").get_view()(); - view_1d omega, divT, divq, u_ls, v_ls, qv_iop, t_iop, u_iop, v_iop; - divT = use_3d_forcing ? m_iop->get_iop_field("divT3d").get_view() - : m_iop->get_iop_field("divT").get_view(); - divq = use_3d_forcing ? m_iop->get_iop_field("divq3d").get_view() - : m_iop->get_iop_field("divq").get_view(); - if (iop_dosubsidence) { - omega = m_iop->get_iop_field("omega").get_view(); - } - if (iop_coriolis) { - u_ls = m_iop->get_iop_field("u_ls").get_view(); - v_ls = m_iop->get_iop_field("v_ls").get_view(); - } - if (iop_nudge_tq) { - qv_iop = m_iop->get_iop_field("q").get_view(); - t_iop = m_iop->get_iop_field("T").get_view(); - } - if (iop_nudge_uv) { - u_iop = use_large_scale_wind ? m_iop->get_iop_field("u_ls").get_view() - : m_iop->get_iop_field("u").get_view(); - v_iop = use_large_scale_wind ? m_iop->get_iop_field("v_ls").get_view() - : m_iop->get_iop_field("v").get_view(); - } - - // Team policy and workspace manager for eamxx - const auto policy_iop = ESU::get_default_team_policy(nelem*NGP*NGP, NLEV); - - // TODO: Create a memory buffer for this class - // and add the below WSM and views - WorkspaceMgr iop_wsm(NLEVI, 7+qsize, policy_iop); - view_Nd - temperature("temperature", nelem, NGP, NGP, NLEV); - - // Lambda for computing temperature - auto compute_temperature = [&] () { - Kokkos::parallel_for("compute_temperature_for_iop", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { - const int ie = team.league_rank()/(NGP*NGP); - const int igp = (team.league_rank()/NGP)%NGP; - const int jgp = team.league_rank()%NGP; - - // Get temp views from workspace - auto ws = iop_wsm.get_workspace(team); - uview_1d pmid; - ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); - - auto ps_i = ps_dyn(ie, igp, jgp); - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - - // Compute reference pressures and layer thickness. - // TODO: Allow geometry data to allocate packsize - auto s_pmid = ekat::scalarize(pmid); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels), [&](const int& k) { - s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; - }); - team.team_barrier(); - - // Compute temperature from virtual potential temperature - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&] (const int k) { - auto T_val = vtheta_dp_i(k); - T_val /= dp3d_i(k); - T_val = PF::calculate_temperature_from_virtual_temperature(T_val,qv_i(k)); - temperature_i(k) = PF::calculate_T_from_theta(T_val,pmid(k)); - }); - - // Release WS views - ws.release_many_contiguous<1>({&pmid}); - }); - }; - - // Preprocess some homme states to get temperature - compute_temperature(); - Kokkos::fence(); - - // Apply IOP forcing - Kokkos::parallel_for("apply_iop_forcing", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { - const int ie = team.league_rank()/(NGP*NGP); - const int igp = (team.league_rank()/NGP)%NGP; - const int jgp = team.league_rank()%NGP; - - // Get temp views from workspace - auto ws = iop_wsm.get_workspace(team); - uview_1d pmid, pint, pdel; - ws.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, - {&pmid, &pint, &pdel}); - - auto ps_i = ps_dyn(ie, igp, jgp); - auto u_i = ekat::subview(v_dyn, ie, 0, igp, jgp); - auto v_i = ekat::subview(v_dyn, ie, 1, igp, jgp); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto Q_i = Kokkos::subview(Q_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); - - // Compute reference pressures and layer thickness. - // TODO: Allow geometry data to allocate packsize - auto s_pmid = ekat::scalarize(pmid); - auto s_pint = ekat::scalarize(pint); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels+1), [&](const int& k) { - s_pint(k) = hyai(k)*ps0 + hybi(k)*ps_i; - if (k < total_levels) { - s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; - } - }); - team.team_barrier(); - ColOps::compute_midpoint_delta(team, total_levels, pint, pdel); - team.team_barrier(); - - if (iop_dosubsidence) { - // Compute subsidence due to large-scale forcing - advance_iop_subsidence(team, total_levels, dt, ps_i, pmid, pint, pdel, omega, ws, u_i, v_i, temperature_i, Q_i); - } - - // Update T and qv according to large scale forcing as specified in IOP file. - advance_iop_forcing(team, total_levels, dt, divT, divq, temperature_i, qv_i); - - if (iop_coriolis) { - // Apply coriolis forcing to u and v winds - iop_apply_coriolis(team, total_levels, dt, lat, u_ls, v_ls, u_i, v_i); - } - - // Release WS views - ws.release_many_contiguous<3>({&pmid, &pint, &pdel}); - }); - Kokkos::fence(); - - // Postprocess homme states Qdp and vtheta_dp - Kokkos::parallel_for("compute_qdp_and_vtheta_dp", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { - const int ie = team.league_rank()/(NGP*NGP); - const int igp = (team.league_rank()/NGP)%NGP; - const int jgp = team.league_rank()%NGP; - - // Get temp views from workspace - auto ws = iop_wsm.get_workspace(team); - uview_1d pmid, pint, pdel; - ws.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, - {&pmid, &pint, &pdel}); - - auto ps_i = ps_dyn(ie, igp, jgp); - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto Q_i = Kokkos::subview(Q_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); - auto Qdp_i = Kokkos::subview(Qdp_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - - // Compute reference pressures and layer thickness. - // TODO: Allow geometry data to allocate packsize - auto s_pmid = ekat::scalarize(pmid); - auto s_pint = ekat::scalarize(pint); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels+1), [&](const int& k) { - s_pint(k) = hyai(k)*ps0 + hybi(k)*ps_i; - if (k < total_levels) { - s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; - } - }); - - team.team_barrier(); - - // Compute Qdp from updated Q - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV*qsize), [&] (const int k) { - const int ilev = k/qsize; - const int q = k%qsize; - - Qdp_i(q, ilev) = Q_i(q, ilev)*dp3d_i(ilev); - // For BFB on restarts, Q needs to be updated after we compute Qdp - Q_i(q, ilev) = Qdp_i(q, ilev)/dp3d_i(ilev); - }); - team.team_barrier(); - - // Convert updated temperature back to psuedo density virtual potential temperature - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&] (const int k) { - const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); - vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); - }); - - // Release WS views - ws.release_many_contiguous<3>({&pmid, &pint, &pdel}); - }); - - if (iop_nudge_tq or iop_nudge_uv) { - // Nudge the domain based on the domain mean - // and observed quantities of T, Q, u, and - - if (iop_nudge_tq) { - // Compute temperature - compute_temperature(); - Kokkos::fence(); - } - - // Compute domain mean of qv, temperature, u, and v - - // TODO: add to local mem buffer - view_1d qv_mean, t_mean, u_mean, v_mean; - if (iop_nudge_tq) { - qv_mean = view_1d("u_mean", NLEV), - t_mean = view_1d("v_mean", NLEV); - } - if (iop_nudge_uv){ - u_mean = view_1d("u_mean", NLEV), - v_mean = view_1d("v_mean", NLEV); - } - - const auto qv_mean_h = Kokkos::create_mirror_view(qv_mean); - const auto t_mean_h = Kokkos::create_mirror_view(t_mean); - const auto u_mean_h = Kokkos::create_mirror_view(u_mean); - const auto v_mean_h = Kokkos::create_mirror_view(v_mean); - - for (int k=0; kget_num_global_dofs(); - t_mean_k /= m_dyn_grid->get_num_global_dofs(); - } - if (iop_nudge_uv){ - Real& u_mean_k = u_mean_h(k/Pack::n)[k%Pack::n]; - Real& v_mean_k = v_mean_h(k/Pack::n)[k%Pack::n]; - Kokkos::parallel_reduce("compute_domain_means_uv", - nelem*NGP*NGP, - KOKKOS_LAMBDA (const int idx, Real& u_sum, Real& v_sum) { - const int ie = idx/(NGP*NGP); - const int igp = (idx/NGP)%NGP; - const int jgp = idx%NGP; - - u_sum += v_dyn(ie, 0, igp, jgp, k/Pack::n)[k%Pack::n]; - v_sum += v_dyn(ie, 1, igp, jgp, k/Pack::n)[k%Pack::n]; - }, - u_mean_k, - v_mean_k); - - m_comm.all_reduce(&u_mean_k, 1, MPI_SUM); - m_comm.all_reduce(&v_mean_k, 1, MPI_SUM); - - u_mean_k /= m_dyn_grid->get_num_global_dofs(); - v_mean_k /= m_dyn_grid->get_num_global_dofs(); - } - } - Kokkos::deep_copy(qv_mean, qv_mean_h); - Kokkos::deep_copy(t_mean, t_mean_h); - Kokkos::deep_copy(u_mean, u_mean_h); - Kokkos::deep_copy(v_mean, v_mean_h); - - // Apply relaxation - const auto rtau = std::max(dt, iop_nudge_tscale); - Kokkos::parallel_for("apply_domain_relaxation", - policy_iop, - KOKKOS_LAMBDA (const KT::MemberType& team) { - - const int ie = team.league_rank()/(NGP*NGP); - const int igp = (team.league_rank()/NGP)%NGP; - const int jgp = team.league_rank()%NGP; - - // Get temp views from workspace - auto ws = iop_wsm.get_workspace(team); - uview_1d pmid; - ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); - - auto ps_i = ps_dyn(ie, igp, jgp); - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - auto u_i = ekat::subview(v_dyn, ie, 0, igp, jgp); - auto v_i = ekat::subview(v_dyn, ie, 1, igp, jgp); - - // Compute reference pressures and layer thickness. - // TODO: Allow geometry data to allocate packsize - auto s_pmid = ekat::scalarize(pmid); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels), [&](const int& k) { - s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; - }); - team.team_barrier(); - - if (iop_nudge_tq or iop_nudge_uv) { - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&](const int& k) { - if (iop_nudge_tq) { - // Restrict nudging of T and qv to certain levels if requested by user - // IOP pressure variable is in unitis of [Pa], while iop_nudge_tq_low/high - // is in units of [hPa], thus convert iop_nudge_tq_low/high - Mask nudge_level(false); - int max_size = hyam.size(); - for (int lev=k*Pack::n, p = 0; p < Pack::n && lev < max_size; ++lev, ++p) { - const auto pressure_from_iop = hyam(lev)*ps0 + hybm(lev)*ps_iop; - nudge_level.set(p, pressure_from_iop <= iop_nudge_tq_low*100 - and - pressure_from_iop >= iop_nudge_tq_high*100); - } - - qv_i(k).update(nudge_level, qv_mean(k) - qv_iop(k), -dt/rtau, 1.0); - temperature_i(k).update(nudge_level, t_mean(k) - t_iop(k), -dt/rtau, 1.0); - - // Convert updated temperature back to virtual potential temperature - const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); - vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); - } - if (iop_nudge_uv) { - u_i(k).update(u_mean(k) - u_iop(k), -dt/rtau, 1.0); - v_i(k).update(v_mean(k) - v_iop(k), -dt/rtau, 1.0); - } - }); - } - - // Release WS views - ws.release_many_contiguous<1>({&pmid}); - }); - } -} - -} // namespace scream diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp index 882261f021b8..a30418c18ee6 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp @@ -172,13 +172,14 @@ void HommeDynamics::set_grids (const std::shared_ptr grids_m add_field("pseudo_density", pg_scalar3d_mid, Pa, pgn,N); add_field("pseudo_density_dry", pg_scalar3d_mid, Pa, pgn,N); add_field ("ps", pg_scalar2d , Pa, pgn); - add_field("qv", pg_scalar3d_mid, kg/kg, pgn,"tracers",N); add_field("phis", pg_scalar2d , m2/s2, pgn); add_field("p_int", pg_scalar3d_int, Pa, pgn,N); add_field("p_mid", pg_scalar3d_mid, Pa, pgn,N); add_field("p_dry_int", pg_scalar3d_int, Pa, pgn,N); add_field("p_dry_mid", pg_scalar3d_mid, Pa, pgn,N); add_field("omega", pg_scalar3d_mid, Pa/s, pgn,N); + + add_tracer("qv", m_phys_grid, kg/kg, N); add_group("tracers",pgn,N, Bundling::Required); if (fv_phys_active()) { @@ -425,11 +426,6 @@ void HommeDynamics::initialize_impl (const RunType run_type) if (run_type==RunType::Initial) { initialize_homme_state (); } else { - if (m_iop) { - // We need to reload IOP data after restarting - m_iop->read_iop_file_data(timestamp()); - } - restart_homme_state (); } @@ -443,7 +439,7 @@ void HommeDynamics::initialize_impl (const RunType run_type) prim_init_model_f90 (); if (fv_phys_active()) { - fv_phys_dyn_to_fv_phys(run_type != RunType::Initial); + fv_phys_dyn_to_fv_phys(run_type == RunType::Restart); // [CGLL ICs in pg2] Remove the CGLL fields from the process. The AD has a // separate fvphyshack-based line to remove the whole CGLL FM. The intention // is to clear the view memory on the device, but I don't know if these two @@ -456,7 +452,7 @@ void HommeDynamics::initialize_impl (const RunType run_type) for (const auto& f : {"horiz_winds", "T_mid", "ps", "phis", "pseudo_density"}) remove_field(f, rgn); remove_group("tracers", rgn); - fv_phys_rrtmgp_active_gases_remap(); + fv_phys_rrtmgp_active_gases_remap(run_type); } // Set up field property checks @@ -668,10 +664,6 @@ void HommeDynamics::homme_post_process (const double dt) { get_internal_field("w_int_dyn").get_header().get_alloc_properties().reset_subview_idx(tl.n0); } - if (m_iop) { - apply_iop_forcing(dt); - } - if (fv_phys_active()) { fv_phys_post_process(); // Apply Rayleigh friction to update temperature and horiz_winds diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp index c6fa150b10b3..b3e01f8aa302 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp @@ -103,50 +103,12 @@ class HommeDynamics : public AtmosphereProcess void fv_phys_post_process(); // See [rrtmgp active gases] in eamxx_homme_fv_phys.cpp. void fv_phys_rrtmgp_active_gases_init(const std::shared_ptr& gm); - void fv_phys_rrtmgp_active_gases_remap(); + void fv_phys_rrtmgp_active_gases_remap (const RunType run_type); // Rayleigh friction functions void rayleigh_friction_init (); void rayleigh_friction_apply (const Real dt) const; - // IOP functions - void apply_iop_forcing(const Real dt); - - KOKKOS_FUNCTION - static void advance_iop_subsidence(const KT::MemberType& team, - const int nlevs, - const Real dt, - const Real ps, - const view_1d& pmid, - const view_1d& pint, - const view_1d& pdel, - const view_1d& omega, - const Workspace& workspace, - const view_1d& u, - const view_1d& v, - const view_1d& T, - const view_2d& Q); - - KOKKOS_FUNCTION - static void advance_iop_forcing(const KT::MemberType& team, - const int nlevs, - const Real dt, - const view_1d& divT, - const view_1d& divq, - const view_1d& T, - const view_1d& qv); - - - KOKKOS_FUNCTION - static void iop_apply_coriolis(const KT::MemberType& team, - const int nlevs, - const Real dt, - const Real lat, - const view_1d& u_ls, - const view_1d& v_ls, - const view_1d& u, - const view_1d& v); - public: // Fast boolean function returning whether Physics PGN is being used. bool fv_phys_active() const; diff --git a/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp b/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp index 87009c7d074e..7376e9af9dde 100644 --- a/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp +++ b/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp @@ -188,7 +188,7 @@ void HommeGridsManager::build_dynamics_grid () { initialize_vertical_coordinates(dyn_grid); dyn_grid->m_short_name = "dyn"; - add_grid(dyn_grid); + add_nonconst_grid(dyn_grid); } void HommeGridsManager:: @@ -271,6 +271,7 @@ build_physics_grid (const ci_string& type, const ci_string& rebalance) { auto hyam = phys_grid->create_geometry_data("hyam",layout_mid,nondim); auto hybm = phys_grid->create_geometry_data("hybm",layout_mid,nondim); auto lev = phys_grid->create_geometry_data("lev", layout_mid,mbar); + auto ilev = phys_grid->create_geometry_data("ilev",layout_int,mbar); for (auto f : {hyai, hybi, hyam, hybm}) { auto f_d = get_grid("Dynamics")->get_geometry_data(f.name()); @@ -281,13 +282,20 @@ build_physics_grid (const ci_string& type, const ci_string& rebalance) { // Build lev from hyam and hybm const Real ps0 = 100000.0; - auto hya_v = hyam.get_view(); - auto hyb_v = hybm.get_view(); - auto lev_v = lev.get_view(); - for (int ii=0;iiget_num_vertical_levels();ii++) { - lev_v(ii) = 0.01*ps0*(hya_v(ii)+hyb_v(ii)); + auto hyam_v = hyam.get_view(); + auto hybm_v = hybm.get_view(); + auto hyai_v = hyai.get_view(); + auto hybi_v = hybi.get_view(); + auto lev_v = lev.get_view(); + auto ilev_v = ilev.get_view(); + auto num_v_levs = phys_grid->get_num_vertical_levels(); + for (int ii=0;iim_short_name = type; - add_grid(phys_grid); + add_nonconst_grid(phys_grid); } void HommeGridsManager:: diff --git a/components/eamxx/src/dynamics/homme/interface/homme_context_mod.F90 b/components/eamxx/src/dynamics/homme/interface/homme_context_mod.F90 index 5e9f6bd2f9d9..377f3c9ed2c2 100644 --- a/components/eamxx/src/dynamics/homme/interface/homme_context_mod.F90 +++ b/components/eamxx/src/dynamics/homme/interface/homme_context_mod.F90 @@ -168,37 +168,37 @@ end subroutine init_parallel_f90 function is_parallel_inited_f90 () result(inited) bind(c) logical (kind=c_bool) :: inited - inited = is_parallel_inited + inited = LOGICAL(is_parallel_inited,kind=c_bool) end function is_parallel_inited_f90 function is_params_inited_f90 () result(inited) bind(c) logical (kind=c_bool) :: inited - inited = is_params_inited + inited = LOGICAL(is_params_inited,kind=c_bool) end function is_params_inited_f90 function is_geometry_inited_f90 () result(inited) bind(c) logical (kind=c_bool) :: inited - inited = is_geometry_inited + inited = LOGICAL(is_geometry_inited,kind=c_bool) end function is_geometry_inited_f90 function is_data_structures_inited_f90 () result(inited) bind(c) logical (kind=c_bool) :: inited - inited = is_data_structures_inited + inited = LOGICAL(is_data_structures_inited,kind=c_bool) end function is_data_structures_inited_f90 function is_model_inited_f90 () result(inited) bind(c) logical (kind=c_bool) :: inited - inited = is_model_inited + inited = LOGICAL(is_model_inited,kind=c_bool) end function is_model_inited_f90 function is_hommexx_functors_inited_f90 () result(inited) bind(c) logical (kind=c_bool) :: inited - inited = is_hommexx_functors_inited + inited = LOGICAL(is_hommexx_functors_inited,kind=c_bool) end function is_hommexx_functors_inited_f90 end module homme_context_mod diff --git a/components/eamxx/src/dynamics/homme/tests/dyn_grid_io.cpp b/components/eamxx/src/dynamics/homme/tests/dyn_grid_io.cpp index ec1e709c7cde..20ca4eb80f0c 100644 --- a/components/eamxx/src/dynamics/homme/tests/dyn_grid_io.cpp +++ b/components/eamxx/src/dynamics/homme/tests/dyn_grid_io.cpp @@ -146,7 +146,6 @@ TEST_CASE("dyn_grid_io") ekat::ParameterList out_params; out_params.set("Averaging Type","Instant"); out_params.set("filename_prefix","dyn_grid_io"); - out_params.set("MPI Ranks in Filename",true); out_params.sublist("Fields").sublist("Dynamics").set>("Field Names",fnames); out_params.sublist("Fields").sublist("Dynamics").set("IO Grid Name","Physics GLL"); @@ -155,7 +154,8 @@ TEST_CASE("dyn_grid_io") out_params.set("Floating Point Precision","real"); OutputManager output; - output.setup (comm, out_params, fm_dyn, gm, t0, t0, false); + output.initialize(comm, out_params, t0, false); + output.setup (fm_dyn, gm); output.run(t0); output.finalize(); diff --git a/components/eamxx/src/mct_coupling/CMakeLists.txt b/components/eamxx/src/mct_coupling/CMakeLists.txt index 308cd1776234..39f864e728aa 100644 --- a/components/eamxx/src/mct_coupling/CMakeLists.txt +++ b/components/eamxx/src/mct_coupling/CMakeLists.txt @@ -38,6 +38,7 @@ set (SCREAM_LIBS eamxx_cosp cld_fraction spa + iop_forcing nudging diagnostics tms diff --git a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 index 5e1b762270b2..0a248a2b6085 100644 --- a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 +++ b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 @@ -95,7 +95,8 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) use ekat_string_utils, only: string_f2c use mct_mod, only: mct_aVect_init, mct_gsMap_lsize use seq_flds_mod, only: seq_flds_a2x_fields, seq_flds_x2a_fields - use seq_infodata_mod, only: seq_infodata_start_type_start, seq_infodata_start_type_cont + use seq_infodata_mod, only: seq_infodata_start_type_start, seq_infodata_start_type_cont, & + seq_infodata_start_type_brnch use seq_comm_mct, only: seq_comm_inst, seq_comm_name, seq_comm_suffix use shr_file_mod, only: shr_file_getunit, shr_file_setIO use shr_sys_mod, only: shr_sys_abort @@ -136,9 +137,9 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) ! TODO: read this from the namelist? character(len=256) :: yaml_fname = "./data/scream_input.yaml" character(kind=c_char,len=256), target :: yaml_fname_c, atm_log_fname_c - character(len=256) :: caseid, username, hostname - character(kind=c_char,len=256), target :: caseid_c, username_c, hostname_c, calendar_c - logical (kind=c_bool) :: restarted_run + character(len=256) :: caseid, username, hostname, rest_caseid, versionid + character(kind=c_char,len=256), target :: caseid_c, username_c, hostname_c, calendar_c, rest_caseid_c, versionid_c + integer (kind=c_int) :: run_type_c !------------------------------------------------------------------------------- ! Grab some data from the cdata structure (coming from the coupler) @@ -149,7 +150,9 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) dom=dom_atm, & infodata=infodata) call seq_infodata_getData(infodata, atm_phase=phase, start_type=run_type, & - username=username, case_name=caseid, hostname=hostname) + username=username, case_name=caseid, & + rest_case_name=rest_caseid, hostname=hostname, & + model_version=versionid) call seq_infodata_PutData(infodata, atm_aero=.true.) call seq_infodata_PutData(infodata, atm_prognostic=.true.) @@ -189,6 +192,17 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) ! Initialize atm !---------------------------------------------------------------------------- + if (trim(run_type) == trim(seq_infodata_start_type_start)) then + run_type_c = 0 + else if (trim(run_type) == trim(seq_infodata_start_type_cont) ) then + run_type_c = 1 + else if (trim(run_type) == trim(seq_infodata_start_type_brnch) ) then + run_type_c = 1 + else + print *, "[eamxx] ERROR! Unsupported run type: "//trim(run_type) + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + ! Init the AD call seq_timemgr_EClockGetData(EClock, calendar=calendar, & curr_ymd=cur_ymd, curr_tod=cur_tod, & @@ -196,11 +210,15 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) call string_f2c(yaml_fname,yaml_fname_c) call string_f2c(calendar,calendar_c) call string_f2c(trim(atm_log_fname),atm_log_fname_c) - call scream_create_atm_instance (mpicom_atm, ATM_ID, yaml_fname_c, atm_log_fname_c, & + call string_f2c(trim(caseid),caseid_c) + call string_f2c(trim(rest_caseid),rest_caseid_c) + call string_f2c(trim(hostname),hostname_c) + call string_f2c(trim(username),username_c) + call string_f2c(trim(versionid),versionid_c) + call scream_create_atm_instance (mpicom_atm, ATM_ID, yaml_fname_c, atm_log_fname_c, run_type_c, & INT(cur_ymd,kind=C_INT), INT(cur_tod,kind=C_INT), & INT(case_start_ymd,kind=C_INT), INT(case_start_tod,kind=C_INT), & - calendar_c) - + calendar_c, caseid_c, rest_caseid_c, hostname_c, username_c, versionid_c) ! Init MCT gsMap call atm_Set_gsMap_mct (mpicom_atm, ATM_ID, gsMap_atm) @@ -221,16 +239,6 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) call mct_aVect_init(x2a, rList=seq_flds_x2a_fields, lsize=lsize) call mct_aVect_init(a2x, rList=seq_flds_a2x_fields, lsize=lsize) - ! Complete AD initialization based on run type - if (trim(run_type) == trim(seq_infodata_start_type_start)) then - restarted_run = .false. - else if (trim(run_type) == trim(seq_infodata_start_type_cont) ) then - restarted_run = .true. - else - print *, "[eamxx] ERROR! Unsupported starttype: "//trim(run_type) - call mpi_abort(mpicom_atm,ierr,mpi_ierr) - endif - ! Init surface coupling stuff in the AD call scream_set_cpl_indices (x2a, a2x) @@ -269,10 +277,7 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) c_loc(export_constant_multiple), c_loc(do_export_during_init), & num_cpl_exports, num_scream_exports, export_field_size) - call string_f2c(trim(caseid),caseid_c) - call string_f2c(trim(username),username_c) - call string_f2c(trim(hostname),hostname_c) - call scream_init_atm (caseid_c,hostname_c,username_c) + call scream_init_atm () #ifdef HAVE_MOAB ! data should be set now inside moab from import and export fields ! do we import and export or just export at init stage ? @@ -308,7 +313,7 @@ subroutine atm_run_mct(EClock, cdata, x2a, a2x) #ifdef MOABCOMP use mct_mod use seq_comm_mct, only : num_moab_exports -#endif +#endif integer :: ent_type #ifdef MOABCOMP @@ -324,7 +329,7 @@ subroutine atm_run_mct(EClock, cdata, x2a, a2x) type(ESMF_Clock) ,intent(inout) :: EClock ! clock type(seq_cdata) ,intent(inout) :: cdata - type(mct_aVect) ,intent(inout) :: x2a ! driver -> atmosphere + type(mct_aVect) ,intent(inout) :: x2a ! driver -> atmosphere type(mct_aVect) ,intent(inout) :: a2x ! atmosphere -> driver !--- local --- @@ -403,7 +408,7 @@ subroutine atm_final_mct(EClock, cdata, x2a, a2x) ! !INPUT/OUTPUT PARAMETERS: type(ESMF_Clock) ,intent(inout) :: EClock ! clock type(seq_cdata) ,intent(inout) :: cdata - type(mct_aVect) ,intent(inout) :: x2a ! driver -> atmosphere + type(mct_aVect) ,intent(inout) :: x2a ! driver -> atmosphere type(mct_aVect) ,intent(inout) :: a2x ! atmosphere -> driver !---------------------------------------------------------------------------- @@ -437,7 +442,7 @@ subroutine atm_Set_gsMap_mct( mpicom_atm, ATMID, GSMap_atm ) ! Build the atmosphere grid numbering for MCT ! NOTE: Numbering scheme is: West to East and South to North ! starting at south pole. Should be the same as what's used in SCRIP - + ! Determine global seg map num_local_cols = scream_get_num_local_cols() num_global_cols = scream_get_num_global_cols() @@ -466,7 +471,7 @@ subroutine atm_domain_mct( lsize, gsMap_atm, dom_atm ) ! integer , intent(in) :: lsize type(mct_gsMap), intent(in) :: gsMap_atm - type(mct_ggrid), intent(inout):: dom_atm + type(mct_ggrid), intent(inout):: dom_atm ! ! Local Variables ! @@ -488,20 +493,20 @@ subroutine atm_domain_mct( lsize, gsMap_atm, dom_atm ) ! Fill in correct values for domain components call scream_get_cols_latlon(c_loc(data1),c_loc(data2)) - call mct_gGrid_importRAttr(dom_atm,"lat",data1,lsize) - call mct_gGrid_importRAttr(dom_atm,"lon",data2,lsize) + call mct_gGrid_importRAttr(dom_atm,"lat",data1,lsize) + call mct_gGrid_importRAttr(dom_atm,"lon",data2,lsize) call scream_get_cols_area(c_loc(data1)) - call mct_gGrid_importRAttr(dom_atm,"area",data1,lsize) + call mct_gGrid_importRAttr(dom_atm,"area",data1,lsize) ! Mask and frac are both exactly 1 data1 = 1.0 - call mct_gGrid_importRAttr(dom_atm,"mask",data1,lsize) - call mct_gGrid_importRAttr(dom_atm,"frac",data1,lsize) + call mct_gGrid_importRAttr(dom_atm,"mask",data1,lsize) + call mct_gGrid_importRAttr(dom_atm,"frac",data1,lsize) ! Aream is computed by mct, so give invalid initial value data1 = -9999.0_R8 - call mct_gGrid_importRAttr(dom_atm,"aream",data1,lsize) + call mct_gGrid_importRAttr(dom_atm,"aream",data1,lsize) end subroutine atm_domain_mct #ifdef HAVE_MOAB diff --git a/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 b/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 index dc5be4de373f..9462750297fb 100644 --- a/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 +++ b/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 @@ -6,7 +6,7 @@ module scream_cpl_indices private ! Focus only on the ones that scream imports/exports (subsets of x2a and a2x) - integer, parameter, public :: num_scream_imports = 19 + integer, parameter, public :: num_scream_imports = 24 integer, parameter, public :: num_scream_exports = 17 integer, public :: num_cpl_imports, num_cpl_exports, import_field_size, export_field_size @@ -92,6 +92,11 @@ subroutine scream_set_cpl_indices (x2a, a2x) import_field_names(17) = 'icefrac' import_field_names(18) = 'fv' import_field_names(19) = 'ram1' + import_field_names(20) = 'sst' + import_field_names(21) = 'dstflx' + import_field_names(22) = 'dstflx' + import_field_names(23) = 'dstflx' + import_field_names(24) = 'dstflx' ! CPL indices import_cpl_indices(1) = mct_avect_indexra(x2a,'Sx_avsdr') @@ -113,10 +118,23 @@ subroutine scream_set_cpl_indices (x2a, a2x) import_cpl_indices(17) = mct_avect_indexra(x2a,'Sf_ifrac') import_cpl_indices(18) = mct_avect_indexra(x2a,'Sl_fv') import_cpl_indices(19) = mct_avect_indexra(x2a,'Sl_ram1') + !sst + import_cpl_indices(20) = mct_avect_indexra(x2a,'So_t') + !dust fluxes + import_cpl_indices(21) = mct_avect_indexra(x2a,'Fall_flxdst1') + import_cpl_indices(22) = mct_avect_indexra(x2a,'Fall_flxdst2') + import_cpl_indices(23) = mct_avect_indexra(x2a,'Fall_flxdst3') + import_cpl_indices(24) = mct_avect_indexra(x2a,'Fall_flxdst4') ! Vector components + !(Faxx_taux and Faxx_tauy) import_vector_components(11) = 0 import_vector_components(12) = 1 + !(dust fluxes) + import_vector_components(21) = 0 + import_vector_components(22) = 1 + import_vector_components(23) = 2 + import_vector_components(24) = 3 ! Constant multiples import_constant_multiple(10) = -1 diff --git a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp index 8bf027e10bfb..8d2edd5ee9ff 100644 --- a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp +++ b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp @@ -101,11 +101,17 @@ extern "C" void scream_create_atm_instance (const MPI_Fint f_comm, const int atm_id, const char* input_yaml_file, const char* atm_log_file, + const int run_type, const int run_start_ymd, const int run_start_tod, const int case_start_ymd, const int case_start_tod, - const char* calendar_name) + const char* calendar_name, + const char* caseid, + const char* rest_caseid, + const char* hostname, + const char* username, + const char* versionid) { using namespace scream; using namespace scream::control; @@ -170,7 +176,9 @@ void scream_create_atm_instance (const MPI_Fint f_comm, const int atm_id, ad.set_comm(atm_comm); ad.set_params(scream_params); ad.init_scorpio(atm_id); - ad.init_time_stamps(run_t0,case_t0); + ad.init_time_stamps(run_t0,case_t0,run_type); + ad.set_provenance_data (caseid,rest_caseid,hostname,username,versionid); + ad.create_output_managers (); ad.create_atm_processes (); ad.create_grids (); ad.create_fields (); @@ -239,9 +247,7 @@ void scream_init_hip_atm () { } #endif -void scream_init_atm (const char* caseid, - const char* hostname, - const char* username) +void scream_init_atm () { using namespace scream; using namespace scream::control; @@ -250,9 +256,6 @@ void scream_init_atm (const char* caseid, // Get the ad, then complete initialization auto& ad = get_ad_nonconst(); - // Set provenance info in the driver (will be added to the output files) - ad.set_provenance_data (caseid,hostname,username); - // Init all fields, atm processes, and output streams ad.initialize_fields (); ad.initialize_atm_procs (); @@ -293,7 +296,7 @@ void scream_finalize (/* args ? */) { // Get the local (i.e., on current atm rank only) number of physics columns int scream_get_num_local_cols () { - int ncols; + int ncols = -1; fpe_guard_wrapper([&]() { const auto& ad = get_ad(); const auto& gm = ad.get_grids_manager(); @@ -307,7 +310,7 @@ int scream_get_num_local_cols () { // Get the global (i.e., the whole earth) number of physics columns int scream_get_num_global_cols () { - int ncols; + int ncols = -1; fpe_guard_wrapper([&]() { const auto& ad = get_ad(); const auto& gm = ad.get_grids_manager(); diff --git a/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 b/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 index 74b45f8330b9..434639e8f823 100644 --- a/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 +++ b/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 @@ -14,17 +14,22 @@ module scream_f2c_mod ! any structure related to the component coupler. Other subroutines ! will have to be called *after* this one, to achieve that. subroutine scream_create_atm_instance (f_comm,atm_id,yaml_fname,atm_log_fname, & + run_type_c, & run_start_ymd,run_start_tod, & case_start_ymd,case_start_tod, & - calendar_name) bind(c) + calendar_name, & + caseid, rest_caseid, hostname, username, & + versionid) bind(c) use iso_c_binding, only: c_int, c_char ! ! Input(s) ! - integer (kind=c_int), value, intent(in) :: f_comm, atm_id + integer (kind=c_int), value, intent(in) :: f_comm, atm_id, run_type_c integer (kind=c_int), value, intent(in) :: run_start_tod, run_start_ymd integer (kind=c_int), value, intent(in) :: case_start_tod, case_start_ymd - character(kind=c_char), target, intent(in) :: yaml_fname(*), atm_log_fname(*), calendar_name(*) + character(kind=c_char), target, intent(in) :: yaml_fname(*), atm_log_fname(*), calendar_name(*), & + caseid(*), rest_caseid(*), hostname(*), username(*), & + versionid(*) end subroutine scream_create_atm_instance subroutine scream_get_cols_latlon (lat, lon) bind(c) @@ -91,9 +96,7 @@ end subroutine scream_init_hip_atm ! During this call, all fields are initialized (i.e., initial conditions are ! loaded), as well as the atm procs (which might use some initial conditions ! to further initialize internal structures), and the output manager. - subroutine scream_init_atm (caseid,hostname,username) bind(c) - use iso_c_binding, only: c_char - character(kind=c_char), target, intent(in) :: caseid(*), hostname(*), username(*) + subroutine scream_init_atm () bind(c) end subroutine scream_init_atm ! This subroutine will run the whole atm model for one atm timestep diff --git a/components/eamxx/src/physics/CMakeLists.txt b/components/eamxx/src/physics/CMakeLists.txt index e0e89e60f80d..f9beda35a20c 100644 --- a/components/eamxx/src/physics/CMakeLists.txt +++ b/components/eamxx/src/physics/CMakeLists.txt @@ -8,8 +8,10 @@ add_subdirectory(p3) if (SCREAM_DOUBLE_PRECISION) add_subdirectory(rrtmgp) add_subdirectory(cosp) + add_subdirectory(tms) + add_subdirectory(iop_forcing) else() - message(STATUS "WARNING: RRTMGP and COSP only supported for double precision builds; skipping") + message(STATUS "WARNING: RRTMGP, COSP, TMS, and IOPForcing only supported for double precision builds; skipping") endif() add_subdirectory(shoc) add_subdirectory(cld_fraction) @@ -21,8 +23,4 @@ add_subdirectory(nudging) if (SCREAM_ENABLE_MAM) add_subdirectory(mam) endif() -if (SCREAM_DOUBLE_PRECISION) - add_subdirectory(tms) -else() - message(STATUS "WARNING: TMS only supported for double precision builds; skipping") -endif() + diff --git a/components/eamxx/src/physics/cld_fraction/eamxx_cld_fraction_process_interface.cpp b/components/eamxx/src/physics/cld_fraction/eamxx_cld_fraction_process_interface.cpp index 6a7b2bf04e8c..b41dcc1f2720 100644 --- a/components/eamxx/src/physics/cld_fraction/eamxx_cld_fraction_process_interface.cpp +++ b/components/eamxx/src/physics/cld_fraction/eamxx_cld_fraction_process_interface.cpp @@ -38,7 +38,7 @@ void CldFraction::set_grids(const std::shared_ptr grids_mana // Set of fields used strictly as input constexpr int ps = Pack::n; - add_field("qi", scalar3d_layout_mid, kg/kg, grid_name,"tracers",ps); + add_tracer("qi", m_grid, kg/kg, ps); add_field("cldfrac_liq", scalar3d_layout_mid, nondim, grid_name,ps); // Set of fields used strictly as output diff --git a/components/eamxx/src/physics/cosp/eamxx_cosp.cpp b/components/eamxx/src/physics/cosp/eamxx_cosp.cpp index 4aa8d53fea8a..5c9b6b36dcb5 100644 --- a/components/eamxx/src/physics/cosp/eamxx_cosp.cpp +++ b/components/eamxx/src/physics/cosp/eamxx_cosp.cpp @@ -49,7 +49,7 @@ void Cosp::set_grids(const std::shared_ptr grids_manager) // Define the different field layouts that will be used for this process - // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and interfaces + // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and interfaces FieldLayout scalar2d = m_grid->get_2d_scalar_layout(); FieldLayout scalar3d_mid = m_grid->get_3d_scalar_layout(true); FieldLayout scalar3d_int = m_grid->get_3d_scalar_layout(false); @@ -71,12 +71,12 @@ void Cosp::set_grids(const std::shared_ptr grids_manager) //add_field("height_mid", scalar3d_mid, m, grid_name); //add_field("height_int", scalar3d_int, m, grid_name); add_field("T_mid", scalar3d_mid, K, grid_name); - add_field("qv", scalar3d_mid, kg/kg, grid_name, "tracers"); - add_field("qc", scalar3d_mid, kg/kg, grid_name, "tracers"); - add_field("qi", scalar3d_mid, kg/kg, grid_name, "tracers"); add_field("phis", scalar2d , m2/s2, grid_name); add_field("pseudo_density", scalar3d_mid, Pa, grid_name); add_field("cldfrac_rad", scalar3d_mid, nondim, grid_name); + add_tracer("qv", m_grid, kg/kg); + add_tracer("qc", m_grid, kg/kg); + add_tracer("qi", m_grid, kg/kg); // Optical properties, should be computed in radiation interface add_field("dtau067", scalar3d_mid, nondim, grid_name); // 0.67 micron optical depth add_field("dtau105", scalar3d_mid, nondim, grid_name); // 10.5 micron optical depth diff --git a/components/eamxx/src/physics/iop_forcing/CMakeLists.txt b/components/eamxx/src/physics/iop_forcing/CMakeLists.txt new file mode 100644 index 000000000000..093ceac73c91 --- /dev/null +++ b/components/eamxx/src/physics/iop_forcing/CMakeLists.txt @@ -0,0 +1,5 @@ +add_library(iop_forcing eamxx_iop_forcing_process_interface.cpp) +target_compile_definitions(iop_forcing PUBLIC EAMXX_HAS_IOP_FORCING) +target_link_libraries(iop_forcing physics_share scream_share) + +target_link_libraries(eamxx_physics INTERFACE iop_forcing) diff --git a/components/eamxx/src/physics/iop_forcing/eamxx_iop_forcing_process_interface.cpp b/components/eamxx/src/physics/iop_forcing/eamxx_iop_forcing_process_interface.cpp new file mode 100644 index 000000000000..c9a3714e1dce --- /dev/null +++ b/components/eamxx/src/physics/iop_forcing/eamxx_iop_forcing_process_interface.cpp @@ -0,0 +1,520 @@ +#include "physics/iop_forcing/eamxx_iop_forcing_process_interface.hpp" + +#include "share/field/field_utils.hpp" +#include "share/property_checks/field_within_interval_check.hpp" + +namespace scream +{ +// ========================================================================================= +void IOPForcing::set_grids(const std::shared_ptr grids_manager) +{ + using namespace ekat::units; + + m_grid = grids_manager->get_grid("Physics"); + const auto& grid_name = m_grid->name(); + + m_num_cols = m_grid->get_num_local_dofs(); // Number of columns on this rank + m_num_levs = m_grid->get_num_vertical_levels(); // Number of levels per column + + // Define the different field layouts that will be used for this process + FieldLayout scalar2d = m_grid->get_2d_scalar_layout(); + FieldLayout scalar3d_mid = m_grid->get_3d_scalar_layout(true); + FieldLayout vector3d_mid = m_grid->get_3d_vector_layout(true,2); + + constexpr int pack_size = Pack::n; + + add_field("ps", scalar2d, Pa, grid_name); + + add_field("horiz_winds", vector3d_mid, m/s, grid_name, pack_size); + add_field("T_mid", scalar3d_mid, K, grid_name, pack_size); + + add_tracer("qv", m_grid, kg/kg, pack_size); + add_group("tracers", grid_name, pack_size, Bundling::Required); + + // Sanity check that iop data manager is setup by driver + EKAT_REQUIRE_MSG(m_iop_data_manager, + "Error! IOPDataManager not setup by driver, but IOPForcing" + "being used as an ATM process.\n"); + + // Create helper fields for finding horizontal means + auto level_only_scalar_layout = scalar3d_mid.clone().strip_dim(0); + auto level_only_vector_layout = vector3d_mid.clone().strip_dim(0); + const auto iop_nudge_tq = m_iop_data_manager->get_params().get("iop_nudge_tq"); + const auto iop_nudge_uv = m_iop_data_manager->get_params().get("iop_nudge_uv"); + if (iop_nudge_tq or iop_nudge_uv) { + create_helper_field("horiz_mean_weights", scalar2d, grid_name, pack_size); + } + if (iop_nudge_tq) { + create_helper_field("qv_mean", level_only_scalar_layout, grid_name, pack_size); + create_helper_field("t_mean", level_only_scalar_layout, grid_name, pack_size); + } + if (iop_nudge_uv) { + create_helper_field("horiz_winds_mean", level_only_vector_layout, grid_name, pack_size); + } +} +// ========================================================================================= +void IOPForcing:: +set_computed_group_impl (const FieldGroup& group) +{ + EKAT_REQUIRE_MSG(group.m_info->size() >= 1, + "Error! IOPForcing requires at least qv as tracer input.\n"); + + const auto& name = group.m_info->m_group_name; + + EKAT_REQUIRE_MSG(name=="tracers", + "Error! IOPForcing was not expecting a field group called '" << name << "\n"); + + EKAT_REQUIRE_MSG(group.m_info->m_bundled, + "Error! IOPForcing expects bundled fields for tracers.\n"); + + m_num_tracers = group.m_info->size(); +} +// ========================================================================================= +size_t IOPForcing::requested_buffer_size_in_bytes() const +{ + // Number of bytes needed by the WorkspaceManager passed to shoc_main + const int nlevi_packs = ekat::npack(m_num_levs+1); + const auto policy = ESU::get_default_team_policy(m_num_cols, nlevi_packs); + const size_t wsm_bytes = WorkspaceMgr::get_total_bytes_needed(nlevi_packs, 7+m_num_tracers, policy); + + return wsm_bytes; +} +// ========================================================================================= +void IOPForcing::init_buffers(const ATMBufferManager &buffer_manager) +{ + EKAT_REQUIRE_MSG(buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), + "Error! Buffers size not sufficient.\n"); + + const int nlevi_packs = ekat::npack(m_num_levs+1); + Pack* mem = reinterpret_cast(buffer_manager.get_memory()); + + // WSM data + m_buffer.wsm_data = mem; + + const auto policy = ESU::get_default_team_policy(m_num_cols, nlevi_packs); + const size_t wsm_npacks = WorkspaceMgr::get_total_bytes_needed(nlevi_packs, 7+m_num_tracers, policy)/sizeof(Pack); + mem += wsm_npacks; + + size_t used_mem = (reinterpret_cast(mem) - buffer_manager.get_memory())*sizeof(Real); + EKAT_REQUIRE_MSG(used_mem==requested_buffer_size_in_bytes(), "Error! Used memory != requested memory for IOPForcing.\n"); +} +// ========================================================================================= +void IOPForcing::create_helper_field (const std::string& name, + const FieldLayout& layout, + const std::string& grid_name, + const int ps) +{ + using namespace ekat::units; + FieldIdentifier id(name,layout,Units::nondimensional(),grid_name); + + // Create the field. Init with NaN's, so we spot instances of uninited memory usage + Field f(id); + f.get_header().get_alloc_properties().request_allocation(ps); + f.allocate_view(); + f.deep_copy(ekat::ScalarTraits::invalid()); + + m_helper_fields[name] = f; +} +// ========================================================================================= +void IOPForcing::initialize_impl (const RunType run_type) +{ + // Set field property checks for the fields in this process + using Interval = FieldWithinIntervalCheck; + add_postcondition_check(get_field_out("T_mid"),m_grid,100.0,500.0,false); + add_postcondition_check(get_field_out("horiz_winds"),m_grid,-400.0,400.0,false); + // For qv, ensure it doesn't get negative, by allowing repair of any neg value. + // TODO: use a repairable lb that clips only "small" negative values + add_postcondition_check(get_field_out("qv"),m_grid,0,0.2,true); + + // Setup WSM for internal local variables + const auto nlevi_packs = ekat::npack(m_num_levs+1); + const auto policy = ESU::get_default_team_policy(m_num_cols, nlevi_packs); + m_workspace_mgr.setup(m_buffer.wsm_data, nlevi_packs, 7+m_num_tracers, policy); + + // Compute field for horizontal contraction weights (1/num_global_dofs) + const auto iop_nudge_tq = m_iop_data_manager->get_params().get("iop_nudge_tq"); + const auto iop_nudge_uv = m_iop_data_manager->get_params().get("iop_nudge_uv"); + const Real one_over_num_dofs = 1.0/m_grid->get_num_global_dofs(); + if (iop_nudge_tq or iop_nudge_uv) m_helper_fields.at("horiz_mean_weights").deep_copy(one_over_num_dofs); +} +// ========================================================================================= +KOKKOS_FUNCTION +void IOPForcing:: +advance_iop_subsidence(const MemberType& team, + const int nlevs, + const Real dt, + const Real ps, + const view_1d& ref_p_mid, + const view_1d& ref_p_int, + const view_1d& ref_p_del, + const view_1d& omega, + const Workspace& workspace, + const view_1d& u, + const view_1d& v, + const view_1d& T, + const view_2d& Q) +{ + constexpr Real Rair = C::Rair; + constexpr Real Cpair = C::Cpair; + + const auto n_q_tracers = Q.extent_int(0); + const auto nlev_packs = ekat::npack(nlevs); + + // Get some temporary views from WS + uview_1d omega_int, delta_u, delta_v, delta_T, tmp; + workspace.take_many_contiguous_unsafe<4>({"omega_int", "delta_u", "delta_v", "delta_T"}, + {&omega_int, &delta_u, &delta_v, &delta_T}); + const auto delta_Q_slot = workspace.take_macro_block("delta_Q", n_q_tracers); + uview_2d delta_Q(delta_Q_slot.data(), n_q_tracers, nlev_packs); + + auto s_ref_p_mid = ekat::scalarize(ref_p_mid); + auto s_omega = ekat::scalarize(omega); + auto s_delta_u = ekat::scalarize(delta_u); + auto s_delta_v = ekat::scalarize(delta_v); + auto s_delta_T = ekat::scalarize(delta_T); + auto s_delta_Q = ekat::scalarize(delta_Q); + auto s_omega_int = ekat::scalarize(omega_int); + + // Compute omega on the interface grid by using a weighted average in pressure + const int pack_begin = 1/Pack::n, pack_end = (nlevs-1)/Pack::n; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, pack_begin, pack_end+1), [&] (const int k){ + auto range_pack = ekat::range(k*Pack::n); + range_pack.set(range_pack<1, 1); + Pack ref_p_mid_k, ref_p_mid_km1, omega_k, omega_km1; + ekat::index_and_shift<-1>(s_ref_p_mid, range_pack, ref_p_mid_k, ref_p_mid_km1); + ekat::index_and_shift<-1>(s_omega, range_pack, omega_k, omega_km1); + + const auto weight = (ref_p_int(k) - ref_p_mid_km1)/(ref_p_mid_k - ref_p_mid_km1); + omega_int(k).set(range_pack>=1 and range_pack<=nlevs-1, + weight*omega_k + (1-weight)*omega_km1); + }); + omega_int(0)[0] = 0; + omega_int(nlevs/Pack::n)[nlevs%Pack::n] = 0; + + // Compute delta views for u, v, T, and Q (e.g., u(k+1) - u(k), k=0,...,nlevs-2) + ColOps::compute_midpoint_delta(team, nlevs-1, u, delta_u); + ColOps::compute_midpoint_delta(team, nlevs-1, v, delta_v); + ColOps::compute_midpoint_delta(team, nlevs-1, T, delta_T); + for (int iq=0; iq(k*Pack::n); + const auto at_top = range_pack==0; + const auto not_at_top = not at_top; + const auto at_bot = range_pack==nlevs-1; + const auto not_at_bot = not at_bot; + const bool any_at_top = at_top.any(); + const bool any_at_bot = at_bot.any(); + + // Get delta(k-1) packs. The range pack should not + // contain index 0 (so that we don't attempt to access + // k=-1 index) or index > nlevs-2 (since delta_* views + // are size nlevs-1). + auto range_pack_for_m1_shift = range_pack; + range_pack_for_m1_shift.set(range_pack<1, 1); + range_pack_for_m1_shift.set(range_pack>nlevs-2, nlevs-2); + Pack delta_u_k, delta_u_km1, + delta_v_k, delta_v_km1, + delta_T_k, delta_T_km1; + ekat::index_and_shift<-1>(s_delta_u, range_pack_for_m1_shift, delta_u_k, delta_u_km1); + ekat::index_and_shift<-1>(s_delta_v, range_pack_for_m1_shift, delta_v_k, delta_v_km1); + ekat::index_and_shift<-1>(s_delta_T, range_pack_for_m1_shift, delta_T_k, delta_T_km1); + + // At the top and bottom of the model, set the end points for + // delta_*_k and delta_*_km1 to be the first and last entries + // of delta_*, respectively. + if (any_at_top) { + delta_u_k.set(at_top, s_delta_u(0)); + delta_v_k.set(at_top, s_delta_v(0)); + delta_T_k.set(at_top, s_delta_T(0)); + } + if (any_at_bot) { + delta_u_km1.set(at_bot, s_delta_u(nlevs-2)); + delta_v_km1.set(at_bot, s_delta_v(nlevs-2)); + delta_T_km1.set(at_bot, s_delta_T(nlevs-2)); + } + + // Get omega_int(k+1) pack. The range pack should not + // contain index > nlevs-1 (since omega_int is size nlevs+1). + auto range_pack_for_p1_shift = range_pack; + range_pack_for_p1_shift.set(range_pack>nlevs-1, nlevs-1); + Pack omega_int_k, omega_int_kp1; + ekat::index_and_shift<1>(s_omega_int, range_pack, omega_int_k, omega_int_kp1); + + const auto fac = (dt/2)/ref_p_del(k); + + // Update u + u(k).update(not_at_bot, fac*omega_int_kp1*delta_u_k, -1, 1); + u(k).update(not_at_top, fac*omega_int_k*delta_u_km1, -1, 1); + + // Update v + v(k).update(not_at_bot, fac*omega_int_kp1*delta_v_k, -1, 1); + v(k).update(not_at_top, fac*omega_int_k*delta_v_km1, -1, 1); + + // Before updating T, first scale using thermal + // expansion term due to LS vertical advection + T(k) *= 1 + (dt*Rair/Cpair)*omega(k)/ref_p_mid(k); + + // Update T + T(k).update(not_at_bot, fac*omega_int_kp1*delta_T_k, -1, 1); + T(k).update(not_at_top, fac*omega_int_k*delta_T_km1, -1, 1); + + // Update Q + Pack delta_tracer_k, delta_tracer_km1; + for (int iq=0; iq(s_delta_tracer, range_pack_for_m1_shift, delta_tracer_k, delta_tracer_km1); + if (any_at_top) delta_tracer_k.set(at_top, s_delta_tracer(0)); + if (any_at_bot) delta_tracer_km1.set(at_bot, s_delta_tracer(nlevs-2)); + + Q(iq, k).update(not_at_bot, fac*omega_int_kp1*delta_tracer_k, -1, 1); + Q(iq, k).update(not_at_top, fac*omega_int_k*delta_tracer_km1, -1, 1); + } + }); + + // Release WS views + workspace.release_macro_block(delta_Q_slot, n_q_tracers); + workspace.release_many_contiguous<4>({&omega_int, &delta_u, &delta_v, &delta_T}); +} +// ========================================================================================= +KOKKOS_FUNCTION +void IOPForcing:: +advance_iop_forcing(const MemberType& team, + const int nlevs, + const Real dt, + const view_1d& divT, + const view_1d& divq, + const view_1d& T, + const view_1d& qv) +{ + const auto nlev_packs = ekat::npack(nlevs); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_packs), [&] (const int k) { + T(k).update(divT(k), dt, 1.0); + qv(k).update(divq(k), dt, 1.0); + }); +} +// ========================================================================================= +KOKKOS_FUNCTION +void IOPForcing:: +iop_apply_coriolis(const MemberType& team, + const int nlevs, + const Real dt, + const Real lat, + const view_1d& u_ls, + const view_1d& v_ls, + const view_1d& u, + const view_1d& v) +{ + constexpr Real pi = C::Pi; + constexpr Real earth_rotation = C::omega; + + // Compute coriolis force + const auto fcor = 2*earth_rotation*std::sin(lat*pi/180); + + const auto nlev_packs = ekat::npack(nlevs); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_packs), [&] (const int k) { + const auto u_cor = v(k) - v_ls(k); + const auto v_cor = u(k) - u_ls(k); + u(k).update(u_cor, dt*fcor, 1.0); + v(k).update(v_cor, -dt*fcor, 1.0); + }); +} +// ========================================================================================= +void IOPForcing::run_impl (const double dt) +{ + // Pack dimensions + const auto nlev_packs = ekat::npack(m_num_levs); + + // Hybrid coord values + const auto ps0 = C::P0; + const auto hyam = m_grid->get_geometry_data("hyam").get_view(); + const auto hybm = m_grid->get_geometry_data("hybm").get_view(); + const auto hyai = m_grid->get_geometry_data("hyai").get_view(); + const auto hybi = m_grid->get_geometry_data("hybi").get_view(); + + // Get FM fields + const auto ps = get_field_in("ps").get_view(); + const auto horiz_winds = get_field_out("horiz_winds").get_view(); + const auto T_mid = get_field_out("T_mid").get_view(); + const auto qv = get_field_out("qv").get_view(); + const auto Q = get_group_out("tracers").m_bundle->get_view(); + + // Load data from IOP files, if necessary + m_iop_data_manager->read_iop_file_data(timestamp()); + + // Define local IOP param values + const auto iop_dosubsidence = m_iop_data_manager->get_params().get("iop_dosubsidence"); + const auto iop_coriolis = m_iop_data_manager->get_params().get("iop_coriolis"); + const auto iop_nudge_tq = m_iop_data_manager->get_params().get("iop_nudge_tq"); + const auto iop_nudge_uv = m_iop_data_manager->get_params().get("iop_nudge_uv"); + const auto use_large_scale_wind = m_iop_data_manager->get_params().get("use_large_scale_wind"); + const auto use_3d_forcing = m_iop_data_manager->get_params().get("use_3d_forcing"); + const auto target_lat = m_iop_data_manager->get_params().get("target_latitude"); + const auto iop_nudge_tscale = m_iop_data_manager->get_params().get("iop_nudge_tscale"); + const auto iop_nudge_tq_low = m_iop_data_manager->get_params().get("iop_nudge_tq_low"); + const auto iop_nudge_tq_high = m_iop_data_manager->get_params().get("iop_nudge_tq_high"); + + // Define local IOP field views + const Real ps_iop = m_iop_data_manager->get_iop_field("Ps").get_view()(); + view_1d omega, divT, divq, u_ls, v_ls, qv_iop, t_iop, u_iop, v_iop; + divT = use_3d_forcing ? m_iop_data_manager->get_iop_field("divT3d").get_view() + : m_iop_data_manager->get_iop_field("divT").get_view(); + divq = use_3d_forcing ? m_iop_data_manager->get_iop_field("divq3d").get_view() + : m_iop_data_manager->get_iop_field("divq").get_view(); + if (iop_dosubsidence) { + omega = m_iop_data_manager->get_iop_field("omega").get_view(); + } + if (iop_coriolis) { + u_ls = m_iop_data_manager->get_iop_field("u_ls").get_view(); + v_ls = m_iop_data_manager->get_iop_field("v_ls").get_view(); + } + if (iop_nudge_tq) { + qv_iop = m_iop_data_manager->get_iop_field("q").get_view(); + t_iop = m_iop_data_manager->get_iop_field("T").get_view(); + } + if (iop_nudge_uv) { + u_iop = use_large_scale_wind ? m_iop_data_manager->get_iop_field("u_ls").get_view() + : m_iop_data_manager->get_iop_field("u").get_view(); + v_iop = use_large_scale_wind ? m_iop_data_manager->get_iop_field("v_ls").get_view() + : m_iop_data_manager->get_iop_field("v").get_view(); + } + + // Team policy and workspace manager for eamxx + const auto policy_iop = ESU::get_default_team_policy(m_num_cols, nlev_packs); + + // Reset internal WSM variables. + m_workspace_mgr.reset_internals(); + + // Avoid implicit capture of this + auto wsm = m_workspace_mgr; + auto num_levs = m_num_levs; + + // Apply IOP forcing + Kokkos::parallel_for("apply_iop_forcing", policy_iop, KOKKOS_LAMBDA (const MemberType& team) { + const int icol = team.league_rank(); + + auto ps_i = ps(icol); + auto u_i = Kokkos::subview(horiz_winds, icol, 0, Kokkos::ALL()); + auto v_i = Kokkos::subview(horiz_winds, icol, 1, Kokkos::ALL()); + auto T_mid_i = ekat::subview(T_mid, icol); + auto qv_i = ekat::subview(qv, icol); + auto Q_i = Kokkos::subview(Q, icol, Kokkos::ALL(), Kokkos::ALL()); + + auto ws = wsm.get_workspace(team); + uview_1d ref_p_mid, ref_p_int, ref_p_del; + ws.take_many_contiguous_unsafe<3>({"ref_p_mid", "ref_p_int", "ref_p_del"}, + {&ref_p_mid, &ref_p_int, &ref_p_del}); + + // Compute reference pressures and layer thickness. + // TODO: Allow geometry data to allocate packsize + auto s_ref_p_mid = ekat::scalarize(ref_p_mid); + auto s_ref_p_int = ekat::scalarize(ref_p_int); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, num_levs+1), [&](const int& k) { + s_ref_p_int(k) = hyai(k)*ps0 + hybi(k)*ps_i; + if (k < num_levs) { + s_ref_p_mid(k) = hyam(k)*ps0 + hybm(k)*ps_i; + } + }); + team.team_barrier(); + ColOps::compute_midpoint_delta(team, num_levs, ref_p_int, ref_p_del); + team.team_barrier(); + + if (iop_dosubsidence) { + // Compute subsidence due to large-scale forcing + advance_iop_subsidence(team, num_levs, dt, ps_i, ref_p_mid, ref_p_int, ref_p_del, omega, ws, u_i, v_i, T_mid_i, Q_i); + } + + // Update T and qv according to large scale forcing as specified in IOP file. + advance_iop_forcing(team, num_levs, dt, divT, divq, T_mid_i, qv_i); + + if (iop_coriolis) { + // Apply coriolis forcing to u and v winds + iop_apply_coriolis(team, num_levs, dt, target_lat, u_ls, v_ls, u_i, v_i); + } + + // Release WS views + ws.release_many_contiguous<3>({&ref_p_mid, &ref_p_int, &ref_p_del}); + }); + + // Nudge the domain based on the domain mean + // and observed quantities of T, Q, u, and v + if (iop_nudge_tq or iop_nudge_uv) { + // Compute domain mean of qv, T_mid, u, and v + view_1d qv_mean, t_mean; + view_2d horiz_winds_mean; + if (iop_nudge_tq){ + horiz_contraction(m_helper_fields.at("qv_mean"), get_field_out("qv"), + m_helper_fields.at("horiz_mean_weights"), &m_comm); + qv_mean = m_helper_fields.at("qv_mean").get_view(); + + horiz_contraction(m_helper_fields.at("t_mean"), get_field_out("T_mid"), + m_helper_fields.at("horiz_mean_weights"), &m_comm); + t_mean = m_helper_fields.at("t_mean").get_view(); + } + if (iop_nudge_uv){ + horiz_contraction(m_helper_fields.at("horiz_winds_mean"), get_field_out("horiz_winds"), + m_helper_fields.at("horiz_mean_weights"), &m_comm); + horiz_winds_mean = m_helper_fields.at("horiz_winds_mean").get_view(); + } + + // Apply relaxation + const auto rtau = std::max(dt, iop_nudge_tscale); + Kokkos::parallel_for("apply_domain_relaxation", + policy_iop, + KOKKOS_LAMBDA (const MemberType& team) { + const int icol = team.league_rank(); + + auto ps_i = ps(icol); + auto u_i = Kokkos::subview(horiz_winds, icol, 0, Kokkos::ALL()); + auto v_i = Kokkos::subview(horiz_winds, icol, 1, Kokkos::ALL()); + auto T_mid_i = ekat::subview(T_mid, icol); + auto qv_i = ekat::subview(qv, icol); + + auto ws = wsm.get_workspace(team); + uview_1d ref_p_mid; + ws.take_many_contiguous_unsafe<1>({"ref_p_mid"},{&ref_p_mid}); + + // Compute reference pressures and layer thickness. + // TODO: Allow geometry data to allocate packsize + auto s_ref_p_mid = ekat::scalarize(ref_p_mid); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, num_levs), [&](const int& k) { + s_ref_p_mid(k) = hyam(k)*ps0 + hybm(k)*ps_i; + }); + team.team_barrier(); + + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_packs), [&](const int& k) { + if (iop_nudge_tq) { + // Restrict nudging of T and qv to certain levels if requested by user + // IOP pressure variable is in unitis of [Pa], while iop_nudge_tq_low/high + // is in units of [hPa], thus convert iop_nudge_tq_low/high + Mask nudge_level(false); + int max_size = hyam.size(); + for (int lev=k*Pack::n, p = 0; p < Pack::n && lev < max_size; ++lev, ++p) { + const auto pressure_from_iop = hyam(lev)*ps0 + hybm(lev)*ps_iop; + nudge_level.set(p, pressure_from_iop <= iop_nudge_tq_low*100 + and + pressure_from_iop >= iop_nudge_tq_high*100); + } + + qv_i(k).update(nudge_level, qv_mean(k) - qv_iop(k), -dt/rtau, 1.0); + T_mid_i(k).update(nudge_level, t_mean(k) - t_iop(k), -dt/rtau, 1.0); + } + if (iop_nudge_uv) { + u_i(k).update(horiz_winds_mean(0, k) - u_iop(k), -dt/rtau, 1.0); + v_i(k).update(horiz_winds_mean(1, k) - v_iop(k), -dt/rtau, 1.0); + } + }); + + // Release WS views + ws.release_many_contiguous<1>({&ref_p_mid}); + }); + } +} +// ========================================================================================= +} // namespace scream diff --git a/components/eamxx/src/physics/iop_forcing/eamxx_iop_forcing_process_interface.hpp b/components/eamxx/src/physics/iop_forcing/eamxx_iop_forcing_process_interface.hpp new file mode 100644 index 000000000000..7cec311a231c --- /dev/null +++ b/components/eamxx/src/physics/iop_forcing/eamxx_iop_forcing_process_interface.hpp @@ -0,0 +1,158 @@ +#ifndef SCREAM_IOP_FORCING_HPP +#define SCREAM_IOP_FORCING_HPP + +#include "ekat/ekat_parameter_list.hpp" +#include "ekat/ekat_workspace.hpp" + +#include "share/atm_process/atmosphere_process.hpp" +#include "share/atm_process/ATMBufferManager.hpp" +#include "share/util/scream_column_ops.hpp" + +#include "physics/share/physics_constants.hpp" + +#include + +namespace scream +{ +/* + * The class responsible for running EAMxx with an intensive + * observation period (IOP). + * + * The AD should store exactly ONE instance of this class stored + * in its list of subcomponents (the AD should make sure of this). + * + * Currently the only use case is the doubly + * periodic model (DP-SCREAM). + */ + +class IOPForcing : public scream::AtmosphereProcess +{ + // Typedefs for process + using KT = ekat::KokkosTypes; + using ESU = ekat::ExeSpaceUtils; + using Pack = ekat::Pack; + using IntPack = ekat::Pack; + using Mask = ekat::Mask; + using WorkspaceMgr = ekat::WorkspaceManager; + using Workspace = WorkspaceMgr::Workspace; + + using MemberType = KT::MemberType; + template + using view_1d = KT::view_1d; + template + using view_2d = KT::view_2d; + template + using uview_1d = ekat::Unmanaged>; + template + using uview_2d = ekat::Unmanaged>; + + using ColOps = ColumnOps; + using C = physics::Constants; + + + +public: + + // Constructors + IOPForcing (const ekat::Comm& comm, const ekat::ParameterList& params) + : AtmosphereProcess(comm, params) {} + + // The type of subcomponent + AtmosphereProcessType type () const { return AtmosphereProcessType::Physics; } + + // The name of the subcomponent + std::string name () const { return "iop"; } + + // Set the grid + void set_grids (const std::shared_ptr grids_manager); + +#ifndef KOKKOS_ENABLE_CUDA + // Cuda requires methods enclosing __device__ lambda's to be public +protected: +#endif + + void initialize_impl (const RunType run_type); + + // Compute effects of large scale subsidence on T, q, u, and v. + KOKKOS_FUNCTION + static void advance_iop_subsidence(const KT::MemberType& team, + const int nlevs, + const Real dt, + const Real ps, + const view_1d& pmid, + const view_1d& pint, + const view_1d& pdel, + const view_1d& omega, + const Workspace& workspace, + const view_1d& u, + const view_1d& v, + const view_1d& T, + const view_2d& Q); + + // Apply large scale forcing for temperature and water vapor as provided by the IOP file + KOKKOS_FUNCTION + static void advance_iop_forcing(const KT::MemberType& team, + const int nlevs, + const Real dt, + const view_1d& divT, + const view_1d& divq, + const view_1d& T, + const view_1d& qv); + + // Provide coriolis forcing to u and v winds, using large scale winds specified in IOP forcing file. + KOKKOS_FUNCTION + static void iop_apply_coriolis(const KT::MemberType& team, + const int nlevs, + const Real dt, + const Real lat, + const view_1d& u_ls, + const view_1d& v_ls, + const view_1d& u, + const view_1d& v); + + void run_impl (const double dt); + +protected: + + void finalize_impl () {} + + // Creates an helper field, not to be shared with the AD's FieldManager + void create_helper_field (const std::string& name, + const FieldLayout& layout, + const std::string& grid_name, + const int ps = 1); + + void set_computed_group_impl (const FieldGroup& group); + + // Computes total number of bytes needed for local variables + size_t requested_buffer_size_in_bytes() const; + + // Set local variables using memory provided by + // the ATMBufferManager + void init_buffers(const ATMBufferManager &buffer_manager); + + // Keep track of field dimensions and other scalar values + // needed in IOP + Int m_num_cols; + Int m_num_levs; + Int m_num_tracers; + + struct Buffer { + Pack* wsm_data; + }; + + // Some helper fields. + std::map m_helper_fields; + + // Struct which contains local variables + Buffer m_buffer; + + // WSM for internal local variables + WorkspaceMgr m_workspace_mgr; + + std::shared_ptr m_grid; +}; // class IOPForcing + +} // namespace scream + +#endif // SCREAM_IOP_FORCING_HPP diff --git a/components/eamxx/src/physics/mam/CMakeLists.txt b/components/eamxx/src/physics/mam/CMakeLists.txt index 9874f79f9eb6..fb6a48733f8a 100644 --- a/components/eamxx/src/physics/mam/CMakeLists.txt +++ b/components/eamxx/src/physics/mam/CMakeLists.txt @@ -42,6 +42,7 @@ add_subdirectory(${EXTERNALS_SOURCE_DIR}/mam4xx ${CMAKE_BINARY_DIR}/externals/ma # EAMxx mam4xx-based atmospheric processes add_library(mam eamxx_mam_microphysics_process_interface.cpp + ${SCREAM_BASE_DIR}/src/physics/rrtmgp/shr_orb_mod_c2f.F90 eamxx_mam_optics_process_interface.cpp eamxx_mam_dry_deposition_process_interface.cpp eamxx_mam_aci_process_interface.cpp @@ -54,7 +55,7 @@ target_include_directories(mam PUBLIC ${EXTERNALS_SOURCE_DIR}/haero ${EXTERNALS_SOURCE_DIR}/mam4xx/src ) -target_link_libraries(mam PUBLIC physics_share scream_share mam4xx haero) +target_link_libraries(mam PUBLIC physics_share csm_share scream_share mam4xx haero) #if (NOT SCREAM_LIB_ONLY) # add_subdirectory(tests) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp index 8bb52c918d23..9dbe97a4ae73 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp @@ -406,6 +406,10 @@ void call_function_dropmixnuc( } }); team.team_barrier(); + // HACK: dropmixnuc() requires the parameter enable_aero_vertical_mix, + // so we define it here until we have a better idea of where it + // might come from + const bool enable_aero_vertical_mix = true; mam4::ndrop::dropmixnuc( team, dt, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), ekat::subview(p_int, icol), ekat::subview(pdel, icol), @@ -417,6 +421,7 @@ void call_function_dropmixnuc( spechygro, lmassptr_amode, num2vol_ratio_min_nmodes, num2vol_ratio_max_nmodes, numptr_amode, nspec_amode, exp45logsig, alogsig, aten, mam_idx, mam_cnst_idx, + enable_aero_vertical_mix, ekat::subview(qcld, icol), // out ekat::subview(wsub, icol), // in ekat::subview(cloud_frac_prev, icol), // in diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index ef2c0f2b2882..449ba4a55d89 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -63,66 +63,62 @@ void MAMAci::set_grids( // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; - // Layout for 3D (2d horiz X 1d vertical) variables - // mid points - FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; + // Layout for 2D (2d horiz) variable + const FieldLayout scalar2d = grid_->get_2d_scalar_layout(); + + // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and // interfaces - FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; + const FieldLayout scalar3d_mid = grid_->get_3d_scalar_layout(true); + const FieldLayout scalar3d_int = grid_->get_3d_scalar_layout(false); // layout for 2D (1d horiz X 1d vertical) variable FieldLayout scalar2d_layout_col{{COL}, {ncol_}}; - auto make_layout = [](const std::vector &extents, - const std::vector &names) { - std::vector tags(extents.size(), CMP); - return FieldLayout(tags, extents, names); - }; - using namespace ekat::units; - auto q_unit = kg / kg; // units of mass mixing ratios of tracers - auto n_unit = 1 / kg; // units of number mixing ratios of tracers + constexpr auto q_unit = kg / kg; // units of mass mixing ratios of tracers + constexpr auto n_unit = 1 / kg; // units of number mixing ratios of tracers - auto nondim = ekat::units::Units::nondimensional(); + constexpr auto nondim = ekat::units::Units::nondimensional(); // atmospheric quantities // specific humidity [kg/kg] - add_field("qv", scalar3d_layout_mid, q_unit, grid_name, "tracers"); + add_tracer("qv", grid_, q_unit); // cloud liquid mass mixing ratio [kg/kg] - add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); + add_tracer("qc", grid_, q_unit); // cloud ice mass mixing ratio [kg/kg] - add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); + add_tracer("qi", grid_, q_unit); // cloud liquid number mixing ratio [1/kg] - add_field("nc", scalar3d_layout_mid, n_unit, grid_name, "tracers"); + add_tracer("nc", grid_, n_unit); // cloud ice number mixing ratio [1/kg] - add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers"); + add_tracer("ni", grid_, n_unit); // Temperature[K] at midpoints - add_field("T_mid", scalar3d_layout_mid, K, grid_name); + add_field("T_mid", scalar3d_mid, K, grid_name); // Vertical pressure velocity [Pa/s] at midpoints - add_field("omega", scalar3d_layout_mid, Pa / s, grid_name); + add_field("omega", scalar3d_mid, Pa / s, grid_name); // Total pressure [Pa] at midpoints - add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); + add_field("p_mid", scalar3d_mid, Pa, grid_name); // Total pressure [Pa] at interfaces - add_field("p_int", scalar3d_layout_int, Pa, grid_name); + add_field("p_int", scalar3d_int, Pa, grid_name); // Layer thickness(pdel) [Pa] at midpoints - add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); + add_field("pseudo_density", scalar3d_mid, Pa, grid_name); // planetary boundary layer height add_field("pbl_height", scalar2d_layout_col, m, grid_name); // cloud fraction [nondimensional] computed by eamxx_cld_fraction_process - add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); + add_field("cldfrac_tot", scalar3d_mid, nondim, grid_name); - auto m2 = pow(m, 2); - auto s2 = pow(s, 2); + constexpr auto m2 = pow(m, 2); + constexpr auto s2 = pow(s, 2); // NOTE: w_variance im microp_aero.F90 in EAM is at "itim_old" dynamics time // step. Since, we are using SE dycore, itim_old is 1 which is equivalent to @@ -130,29 +126,30 @@ void MAMAci::set_grids( // and we might need to revisit this. // Vertical velocity variance at midpoints - add_field("w_variance", scalar3d_layout_mid, m2 / s2, grid_name); + add_field("w_variance", scalar3d_mid, m2 / s2, grid_name); // NOTE: "cldfrac_liq" is updated in SHOC. "cldfrac_liq" in C++ code is // equivalent to "alst" in the shoc_intr.F90. In the C++ code, it is used as // "shoc_cldfrac" and in the F90 code it is called "cloud_frac" // Liquid stratiform cloud fraction at midpoints - add_field("cldfrac_liq", scalar3d_layout_mid, nondim, grid_name); + add_field("cldfrac_liq", scalar3d_mid, nondim, grid_name); // Previous value of liquid stratiform cloud fraction at midpoints - add_field("cldfrac_liq_prev", scalar3d_layout_mid, nondim, - grid_name); + add_field("cldfrac_liq_prev", scalar3d_mid, nondim, grid_name); // Eddy diffusivity for heat - add_field("eddy_diff_heat", scalar3d_layout_mid, m2 / s, grid_name); + add_field("eddy_diff_heat", scalar3d_mid, m2 / s, grid_name); - // Layout for 4D (2d horiz X 1d vertical x number of modes) variables - const int num_aero_modes = mam_coupling::num_aero_modes(); - FieldLayout scalar4d_layout_mid = - make_layout({ncol_, num_aero_modes, nlev_}, {"COL", "NMODES", "LEV"}); + // Number of modes + constexpr int nmodes = mam4::AeroConfig::num_modes(); + + // layout for 3D (ncol, nmodes, nlevs) + FieldLayout scalar3d_mid_nmodes = + grid_->get_3d_vector_layout(true, nmodes, "nmodes"); // dry diameter of aerosols [m] - add_field("dgnum", scalar4d_layout_mid, m, grid_name); + add_field("dgnum", scalar3d_mid_nmodes, m, grid_name); // ======================================================================== // Output from this whole process @@ -164,24 +161,21 @@ void MAMAci::set_grids( // interstitial aerosol tracers of interest: number (n) mixing ratios const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(mode); - add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, - grid_name, "tracers"); + add_tracer(int_nmr_field_name, grid_, n_unit); // cloudborne aerosol tracers of interest: number (n) mixing ratios // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are // NOT advected const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(mode); - add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, - grid_name); + add_field(cld_nmr_field_name, scalar3d_mid, n_unit, grid_name); for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(mode, a); if(strlen(int_mmr_field_name) > 0) { - add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, - grid_name, "tracers"); + add_tracer(int_mmr_field_name, grid_, q_unit); } // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are @@ -189,16 +183,14 @@ void MAMAci::set_grids( const char *cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(mode, a); if(strlen(cld_mmr_field_name) > 0) { - add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, - grid_name); + add_field(cld_mmr_field_name, scalar3d_mid, q_unit, grid_name); } } // end for loop num species } // end for loop for num modes for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, - grid_name, "tracers"); + add_tracer(gas_mmr_field_name, grid_, q_unit); } // end for loop num gases // ------------------------------------------------------------------------ @@ -206,54 +198,52 @@ void MAMAci::set_grids( // ------------------------------------------------------------------------ // number of activated aerosol for ice nucleation[#/kg] - add_field("ni_activated", scalar3d_layout_mid, n_unit, grid_name); + add_field("ni_activated", scalar3d_mid, n_unit, grid_name); // ------------------------------------------------------------------------ // Output from droplet activation process (dropmixnuc) // ------------------------------------------------------------------------ // tendency in droplet number mixing ratio [#/kg/s] - add_field("nc_nuceat_tend", scalar3d_layout_mid, n_unit / s, - grid_name); + add_field("nc_nuceat_tend", scalar3d_mid, n_unit / s, grid_name); // FIXME: [TEMPORARY]droplet number mixing ratio source tendency [#/kg/s] - add_field("nsource", scalar3d_layout_mid, n_unit / s, grid_name); + add_field("nsource", scalar3d_mid, n_unit / s, grid_name); // FIXME: [TEMPORARY]droplet number mixing ratio tendency due to mixing // [#/kg/s] - add_field("ndropmix", scalar3d_layout_mid, n_unit / s, grid_name); + add_field("ndropmix", scalar3d_mid, n_unit / s, grid_name); // FIXME: [TEMPORARY]droplet number as seen by ACI [#/kg] - add_field("nc_inp_to_aci", scalar3d_layout_mid, n_unit / s, - grid_name); - const auto cm_tmp = m / 100; // FIXME: [TEMPORARY] remove this - const auto cm3 = cm_tmp * cm_tmp * cm_tmp; // FIXME: [TEMPORARY] remove this + add_field("nc_inp_to_aci", scalar3d_mid, n_unit / s, grid_name); + constexpr auto cm_tmp = m / 100; // FIXME: [TEMPORARY] remove this + constexpr auto cm3 = pow(cm_tmp, 3); // FIXME: [TEMPORARY] remove this // FIXME: [TEMPORARY] remove the following ccn outputs - add_field("ccn_0p02", scalar3d_layout_mid, cm3, grid_name); - add_field("ccn_0p05", scalar3d_layout_mid, cm3, grid_name); - add_field("ccn_0p1", scalar3d_layout_mid, cm3, grid_name); - add_field("ccn_0p2", scalar3d_layout_mid, cm3, grid_name); - add_field("ccn_0p5", scalar3d_layout_mid, cm3, grid_name); - add_field("ccn_1p0", scalar3d_layout_mid, cm3, grid_name); + add_field("ccn_0p02", scalar3d_mid, cm3, grid_name); + add_field("ccn_0p05", scalar3d_mid, cm3, grid_name); + add_field("ccn_0p1", scalar3d_mid, cm3, grid_name); + add_field("ccn_0p2", scalar3d_mid, cm3, grid_name); + add_field("ccn_0p5", scalar3d_mid, cm3, grid_name); + add_field("ccn_1p0", scalar3d_mid, cm3, grid_name); // ------------------------------------------------------------------------ // Output from hetrozenous freezing // ------------------------------------------------------------------------ - const auto cm = m / 100; + constexpr auto cm = m / 100; // units of number mixing ratios of tracers - auto frz_unit = 1 / (cm * cm * cm * s); + constexpr auto frz_unit = 1 / (cm * cm * cm * s); // heterogeneous freezing by immersion nucleation [cm^-3 s^-1] - add_field("hetfrz_immersion_nucleation_tend", scalar3d_layout_mid, + add_field("hetfrz_immersion_nucleation_tend", scalar3d_mid, frz_unit, grid_name); // heterogeneous freezing by contact nucleation [cm^-3 s^-1] - add_field("hetfrz_contact_nucleation_tend", scalar3d_layout_mid, - frz_unit, grid_name); + add_field("hetfrz_contact_nucleation_tend", scalar3d_mid, frz_unit, + grid_name); // heterogeneous freezing by deposition nucleation [cm^-3 s^-1] - add_field("hetfrz_deposition_nucleation_tend", scalar3d_layout_mid, + add_field("hetfrz_deposition_nucleation_tend", scalar3d_mid, frz_unit, grid_name); } // function set_grids ends @@ -302,11 +292,11 @@ void MAMAci::initialize_impl(const RunType run_type) { // store rest fo the atm fields in dry_atm_in dry_atm_.z_surf = 0; - dry_atm_.T_mid = get_field_in("T_mid").get_view(); - dry_atm_.p_mid = get_field_in("p_mid").get_view(); - dry_atm_.p_int = get_field_in("p_int").get_view(); - dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - dry_atm_.omega = get_field_in("omega").get_view(); + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_int = get_field_in("p_int").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.omega = get_field_in("omega").get_view(); // store fields converted to dry mmr from wet mmr in dry_atm_ dry_atm_.qv = buffer_.qv_dry; @@ -564,7 +554,8 @@ void MAMAci::run_impl(const double dt) { // output w0_, rho_); - compute_tke_at_interfaces(team_policy, w_sec_mid_, dry_atm_.dz, nlev_, w_sec_int_, + compute_tke_at_interfaces(team_policy, w_sec_mid_, dry_atm_.dz, nlev_, + w_sec_int_, // output tke_); @@ -597,25 +588,26 @@ void MAMAci::run_impl(const double dt) { // output cloud_frac_, cloud_frac_prev_); - mam_coupling::compute_recipical_pseudo_density(team_policy, dry_atm_.p_del, nlev_, - // output - rpdel_); + mam_coupling::compute_recipical_pseudo_density(team_policy, dry_atm_.p_del, + nlev_, + // output + rpdel_); Kokkos::fence(); // wait for rpdel_ to be computed. // Compute activated CCN number tendency (tendnd_) and updated // cloud borne aerosols (stored in a work array) and interstitial // aerosols tendencies - call_function_dropmixnuc(team_policy, dt, dry_atm_, rpdel_, kvh_mid_, kvh_int_, wsub_, - cloud_frac_, cloud_frac_prev_, dry_aero_, nlev_, - // output - coltend_, coltend_cw_, qcld_, ndropcol_, ndropmix_, - nsource_, wtke_, ccn_, - // ## output to be used by the other processes ## - qqcw_fld_work_, ptend_q_, factnum_, tendnd_, - // work arrays - raercol_cw_, raercol_, state_q_work_, nact_, mact_, - dropmixnuc_scratch_mem_); + call_function_dropmixnuc( + team_policy, dt, dry_atm_, rpdel_, kvh_mid_, kvh_int_, wsub_, cloud_frac_, + cloud_frac_prev_, dry_aero_, nlev_, + // output + coltend_, coltend_cw_, qcld_, ndropcol_, ndropmix_, nsource_, wtke_, ccn_, + // ## output to be used by the other processes ## + qqcw_fld_work_, ptend_q_, factnum_, tendnd_, + // work arrays + raercol_cw_, raercol_, state_q_work_, nact_, mact_, + dropmixnuc_scratch_mem_); Kokkos::fence(); // wait for ptend_q_ to be computed. Kokkos::deep_copy(ccn_0p02_, diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 440e6e3dbdca..2428dd5d9ea7 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -43,19 +43,19 @@ void MAMConstituentFluxes::set_grids( // -------------------------------------------------------------------------- // ----------- Atmospheric quantities ------------- // Specific humidity [kg/kg](Require only for building DS) - add_field("qv", scalar3d_mid, q_unit, grid_name, "tracers"); + add_tracer("qv", grid_, q_unit); // Cloud liquid mass mixing ratio [kg/kg](Require only for building DS) - add_field("qc", scalar3d_mid, q_unit, grid_name, "tracers"); + add_tracer("qc", grid_, q_unit); // Cloud ice mass mixing ratio [kg/kg](Require only for building DS) - add_field("qi", scalar3d_mid, q_unit, grid_name, "tracers"); + add_tracer("qi", grid_, q_unit); // Cloud liquid number mixing ratio [1/kg](Require only for building DS) - add_field("nc", scalar3d_mid, n_unit, grid_name, "tracers"); + add_tracer("nc", grid_, n_unit); // Cloud ice number mixing ratio [1/kg](Require only for building DS) - add_field("ni", scalar3d_mid, n_unit, grid_name, "tracers"); + add_tracer("ni", grid_, n_unit); // Temperature[K] at midpoints add_field("T_mid", scalar3d_mid, K, grid_name); @@ -102,8 +102,7 @@ void MAMConstituentFluxes::set_grids( // interstitial aerosol tracers of interest: number (n) mixing ratios const std::string int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(mode); - add_field(int_nmr_field_name, scalar3d_mid, n_unit, grid_name, - "tracers"); + add_tracer(int_nmr_field_name, grid_, n_unit); // cloudborne aerosol tracers of interest: number (n) mixing ratios // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are @@ -117,8 +116,7 @@ void MAMConstituentFluxes::set_grids( const std::string int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(mode, a); if(not int_mmr_field_name.empty()) { - add_field(int_mmr_field_name, scalar3d_mid, q_unit, grid_name, - "tracers"); + add_tracer(int_mmr_field_name, grid_, q_unit); } // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are @@ -133,8 +131,7 @@ void MAMConstituentFluxes::set_grids( for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const std::string gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - add_field(gas_mmr_field_name, scalar3d_mid, q_unit, grid_name, - "tracers"); + add_tracer(gas_mmr_field_name, grid_, q_unit); } // end for loop num gases } // set_grid diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index f5423e8eb1cf..93c085c8c1f4 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -19,7 +19,7 @@ void compute_tendencies( const MAMDryDep::const_view_1d ocnfrac, const MAMDryDep::const_view_1d friction_velocity, const MAMDryDep::const_view_1d aerodynamical_resistance, - MAMDryDep::view_3d qtracers, MAMDryDep::view_2d fraction_landuse_, + const MAMDryDep::const_view_2d fraction_landuse_, const MAMDryDep::const_view_3d dgncur_awet_, const MAMDryDep::const_view_3d wet_dens_, const mam_coupling::DryAtmosphere dry_atm, @@ -35,7 +35,7 @@ void compute_tendencies( // work arrays MAMDryDep::view_2d rho_, MAMDryDep::view_4d vlc_dry_, MAMDryDep::view_3d vlc_trb_, MAMDryDep::view_4d vlc_grv_, - MAMDryDep::view_3d dqdt_tmp_) { + MAMDryDep::view_3d dqdt_tmp_, MAMDryDep::view_3d qtracers) { static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); const auto policy = ekat::ExeSpaceUtils::get_default_team_policy( @@ -86,7 +86,7 @@ void compute_tendencies( static constexpr int n_land_type = MAMDryDep::n_land_type; Real fraction_landuse[n_land_type]; for(int i = 0; i < n_land_type; ++i) { - fraction_landuse[i] = fraction_landuse_(i, icol); + fraction_landuse[i] = fraction_landuse_(icol, i); } static constexpr int nmodes = mam4::AeroConfig::num_modes(); @@ -181,24 +181,6 @@ void update_cloudborne_mmrs(const MAMDryDep::view_3d qqcw, const double dt, } } // Update cloud borne aerosols ends -// FIXME: remove the following function after implementing file read for landuse -void populated_fraction_landuse(MAMDryDep::view_2d flu, const int ncol) { - Kokkos::parallel_for( - "populated_fraction_landuse", 1, KOKKOS_LAMBDA(int) { - static constexpr int n_land_type = MAMDryDep::n_land_type; - const Real temp[n_land_type] = { - 0.28044346587077795E-003, 0.26634987180780171E-001, - 0.16803558403621365E-001, 0.18076055155371872E-001, - 0.00000000000000000E+000, 0.00000000000000000E+000, - 0.91803784897907303E+000, 0.17186036997038400E-002, - 0.00000000000000000E+000, 0.00000000000000000E+000, - 0.18448503115578840E-001}; - for(int i = 0; i < n_land_type; ++i) - for(int j = 0; j < ncol; ++j) flu(i, j) = temp[i]; - }); - Kokkos::fence(); -} - } // namespace } // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 034972f85258..1b4ad91d0681 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -3,16 +3,13 @@ // Drydep functions are stored in the following hpp file #include -/* ------------------------------------------------------------------ -NOTES: -1. Add a CIME test and multi-process tests -2. Ensure that the submodule for MAM4xx is the main branch -3. Read file for fractional landuse ------------------------------------------------------------------ -*/ +// For reading fractional land use file +#include + namespace scream { +using FracLandUseFunc = frac_landuse::fracLandUseFunctions; + MAMDryDep::MAMDryDep(const ekat::Comm &comm, const ekat::ParameterList ¶ms) : AtmosphereProcess(comm, params) { /* Anything that can be initialized without grid information can be @@ -56,7 +53,8 @@ void MAMDryDep::set_grids( // Layout for 4D (2d horiz X 1d vertical x number of modes) variables // at mid points const int num_aero_modes = mam_coupling::num_aero_modes(); - const FieldLayout vector3d_mid = grid_->get_3d_vector_layout(true, num_aero_modes, "num_modes"); + const FieldLayout vector3d_mid = + grid_->get_3d_vector_layout(true, num_aero_modes, "num_modes"); using namespace ekat::units; @@ -73,19 +71,19 @@ void MAMDryDep::set_grids( // ----------- Atmospheric quantities ------------- // Specific humidity [kg/kg](Require only for building DS) - add_field("qv", scalar3d_mid, q_unit, grid_name, "tracers"); + add_tracer("qv", grid_, q_unit); // Cloud liquid mass mixing ratio [kg/kg](Require only for building DS) - add_field("qc", scalar3d_mid, q_unit, grid_name, "tracers"); + add_tracer("qc", grid_, q_unit); // Cloud ice mass mixing ratio [kg/kg](Require only for building DS) - add_field("qi", scalar3d_mid, q_unit, grid_name, "tracers"); + add_tracer("qi", grid_, q_unit); // Cloud liquid number mixing ratio [1/kg](Require only for building DS) - add_field("nc", scalar3d_mid, n_unit, grid_name, "tracers"); + add_tracer("nc", grid_, n_unit); // Cloud ice number mixing ratio [1/kg](Require only for building DS) - add_field("ni", scalar3d_mid, n_unit, grid_name, "tracers"); + add_tracer("ni", grid_, n_unit); // Temperature[K] at midpoints add_field("T_mid", scalar3d_mid, K, grid_name); @@ -158,15 +156,13 @@ void MAMDryDep::set_grids( for(int m = 0; m < num_aero_modes; ++m) { const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); - add_field(int_nmr_field_name, scalar3d_mid, n_unit, grid_name, - "tracers"); + add_tracer(int_nmr_field_name, grid_, n_unit); for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); if(strlen(int_mmr_field_name) > 0) { - add_field(int_mmr_field_name, scalar3d_mid, q_unit, grid_name, - "tracers"); + add_tracer(int_mmr_field_name, grid_, q_unit); } } } @@ -188,8 +184,7 @@ void MAMDryDep::set_grids( // aerosol-related gases: mass mixing ratios for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - add_field(gas_mmr_field_name, scalar3d_mid, q_unit, grid_name, - "tracers"); + add_tracer(gas_mmr_field_name, grid_, q_unit); } // ------------------------------------------------------------- @@ -202,6 +197,27 @@ void MAMDryDep::set_grids( // surface deposition flux of interstitial aerosols, [kg/m2/s] or [1/m2/s] add_field("deposition_flux_of_interstitial_aerosols", vector2d_pcnst, 1 / m2 / s, grid_name); + + // ------------------------------------------------------------- + // setup to enable reading fractional land use file + // ------------------------------------------------------------- + + const auto mapping_file = m_params.get("drydep_remap_file", ""); + const std::string frac_landuse_data_file = + m_params.get("fractional_land_use_file"); + + // Field to be read from file + const std::string field_name = "fraction_landuse"; + + // Dimensions of the filed + const std::string dim_name1 = "ncol"; + const std::string dim_name2 = "class"; + + // initialize the file read + FracLandUseFunc::init_frac_landuse_file_read( + ncol_, field_name, dim_name1, dim_name2, grid_, frac_landuse_data_file, + mapping_file, horizInterp_, dataReader_); // output + } // set_grids // ================================================================ @@ -348,10 +364,14 @@ void MAMDryDep::initialize_impl(const RunType run_type) { // Work array to hold tendency for 1 species [kg/kg/s] or [1/kg/s] dqdt_tmp_ = view_3d("dqdt_tmp_", pcnst, ncol_, nlev_); - static constexpr int n_land_type = mam4::DryDeposition::n_land_type; - // FIXME: This should come from a file reading - // The fraction of land use for the column. [non-dimentional] - fraction_landuse_ = view_2d("fraction_landuse_", n_land_type, ncol_); + //----------------------------------------------------------------- + // Read fractional land use data + //----------------------------------------------------------------- + // This data is time-independent, we read all data here for the + // entire simulation + FracLandUseFunc::update_frac_land_use_data_from_file( + dataReader_, *horizInterp_, + frac_landuse_); // output //----------------------------------------------------------------- // Setup preprocessing and post processing @@ -406,21 +426,20 @@ void MAMDryDep::run_impl(const double dt) { auto aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") .get_view(); - // FIXME: remove it if it read from a file - populated_fraction_landuse(fraction_landuse_, ncol_); - + //-------------------------------------------------------------------- // Call drydeposition and get tendencies + //-------------------------------------------------------------------- compute_tendencies(ncol_, nlev_, dt, obukhov_length_, surface_friction_velocty_, land_fraction_, ice_fraction_, ocean_fraction_, friction_velocity_, - aerodynamical_resistance_, qtracers_, fraction_landuse_, - dgncur_awet_, wet_dens_, dry_atm_, dry_aero_, + aerodynamical_resistance_, frac_landuse_, dgncur_awet_, + wet_dens_, dry_atm_, dry_aero_, // Inouts-outputs qqcw_, // Outputs ptend_q_, aerdepdrycw_, aerdepdryis_, // work arrays - rho_, vlc_dry_, vlc_trb_, vlc_grv_, dqdt_tmp_); + rho_, vlc_dry_, vlc_trb_, vlc_grv_, dqdt_tmp_, qtracers_); Kokkos::fence(); // Update the interstitial aerosols using ptend. diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index e39090cecdea..55ab5b15b955 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -7,6 +7,9 @@ // For MAM4 aerosol configuration #include +// For AtmosphereInput +#include "share/io/scorpio_input.hpp" + // For component name #include @@ -26,6 +29,7 @@ class MAMDryDep final : public scream::AtmosphereProcess { using view_3d = Field::view_dev_t; using view_4d = Field::view_dev_t; using const_view_1d = Field::view_dev_t; + using const_view_2d = Field::view_dev_t; using const_view_3d = Field::view_dev_t; private: @@ -45,7 +49,6 @@ class MAMDryDep final : public scream::AtmosphereProcess { // physics grid for column information std::shared_ptr grid_; - /* Note on mam4::DryDeposition::aerosol_categories = 4 used in deposition velocity dimension defined below. These correspond to the two attachment states and two moments: @@ -64,14 +67,13 @@ class MAMDryDep final : public scream::AtmosphereProcess { // Dimensions // [num_modes, aerosol_categories_, num columns, num levels] view_4d vlc_grv_; - + // Output deposition velocity, [m/s] - // fraction landuse weighted sum of vlc_grv and vlc_trb + // fraction landuse weighted sum of vlc_grv and vlc_trb // Dimensions // [num_modes, aerosol_categories_, num columns, num levels] view_4d vlc_dry_; - // Output of the the mixing ratio tendencies [kg/kg/s or 1/kg/s] // Dimensions // [num columns, num levels, mam4::aero_model::pcnst] @@ -81,25 +83,18 @@ class MAMDryDep final : public scream::AtmosphereProcess { // Work array to hold the mixing ratios [kg/kg or 1/kg] // Dimensions // [num columns, num levels, mam4::aero_model::pcnst] - // Packs AerosolState::int_aero_nmr + // Packs AerosolState::int_aero_nmr // and AerosolState::int_aero_nmr // into one array, hence is mixed kg/kg and 1/kg. view_3d qtracers_; - // Work array to hold the fraction [non-dimentional] - // of land use for column. - // Dimensions - // [MAMDryDep::n_land_type, num columns] - // Values should sum to 1. - view_2d fraction_landuse_; - // Work array to hold the air density [kg/m3] // Dimensions // [num columns, num levels] // Calculated from air pressure at layer midpoint, // Constants::r_gas_dry_air and air temperture. view_2d rho_; - + // Work array to hold tendency for 1 species [kg/kg/s] or [1/kg/s] // Dimensions // [mam4::aero_model::pcnst, num column, num level] @@ -111,6 +106,11 @@ class MAMDryDep final : public scream::AtmosphereProcess { // Filled with Prognostics::n_mode_c and Prognostics::q_aero_c view_3d qqcw_; + // For reading fractional land use file + std::shared_ptr horizInterp_; + std::shared_ptr dataReader_; + const_view_2d frac_landuse_; + public: using KT = ekat::KokkosTypes; diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index f8a34bd90d5b..1c0053a87b94 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -1,215 +1,464 @@ +#include #include -#include -#include -#include +// impl namespace for some driver level functions for microphysics -#include "scream_config.h" // for SCREAM_CIME_BUILD +#include "physics/rrtmgp/shr_orb_mod_c2f.hpp" +#include "readfiles/find_season_index_utils.hpp" +#include "readfiles/photo_table_utils.cpp" -#include // for serial NetCDF file reads on MPI root -#include +namespace scream { -// NOTE: see the impl/ directory for the contents of the impl namespace -#include "impl/compute_o3_column_density.cpp" -#include "impl/compute_water_content.cpp" -#include "impl/gas_phase_chemistry.cpp" +MAMMicrophysics::MAMMicrophysics(const ekat::Comm &comm, + const ekat::ParameterList ¶ms) + : AtmosphereProcess(comm, params), aero_config_() { + config_.amicphys.do_cond = m_params.get("mam4_do_cond"); + config_.amicphys.do_rename = m_params.get("mam4_do_rename"); + config_.amicphys.do_newnuc = m_params.get("mam4_do_newnuc"); + config_.amicphys.do_coag = m_params.get("mam4_do_coag"); -namespace scream -{ + // these parameters guide the coupling between parameterizations + // NOTE: mam4xx was ported with these parameters fixed, so it's probably not + // NOTE: safe to change these without code modifications. + + // controls treatment of h2so4 condensation in mam_gasaerexch_1subarea + // 1 = sequential calc. of gas-chem prod then condensation loss + // 2 = simultaneous calc. of gas-chem prod and condensation loss + config_.amicphys.gaexch_h2so4_uptake_optaa = 2; + + // controls treatment of h2so4 concentrationin mam_newnuc_1subarea + // 1 = use average value calculated in standard cam5.2.10 and earlier + // 2 = use average value calculated in mam_gasaerexch_1subarea + // 11 = use average of initial and final values from mam_gasaerexch_1subarea + // 12 = use final value from mam_gasaerexch_1subarea + config_.amicphys.newnuc_h2so4_conc_optaa = 2; -MAMMicrophysics::MAMMicrophysics( - const ekat::Comm& comm, - const ekat::ParameterList& params) - : AtmosphereProcess(comm, params), - aero_config_() { - configure(params); + // LINOZ namelist parameters + config_.linoz.o3_lbl = m_params.get("mam4_o3_lbl"); + config_.linoz.o3_tau = m_params.get("mam4_o3_tau"); + config_.linoz.o3_sfc = m_params.get("mam4_o3_sfc"); + config_.linoz.psc_T = m_params.get("mam4_psc_T"); } AtmosphereProcessType MAMMicrophysics::type() const { return AtmosphereProcessType::Physics; } -std::string MAMMicrophysics::name() const { - return "mam4_micro"; -} +// ================================================================ +// SET_GRIDS +// ================================================================ +void MAMMicrophysics::set_grids( + const std::shared_ptr grids_manager) { + // set grid for all the inputs and outputs + // use physics grid + grid_ = grids_manager->get_grid("Physics"); + const auto &grid_name = grid_->name(); -namespace { + ncol_ = grid_->get_num_local_dofs(); // number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // number of levels per column -void set_data_file(const char *name, const char *path, char location[MAX_FILENAME_LEN]) { - EKAT_REQUIRE_MSG(strlen(SCREAM_DATA_DIR) + strlen(path) < MAX_FILENAME_LEN, - "Error! " << name << " path is too long (must be < " << MAX_FILENAME_LEN << " characters)"); - sprintf(location, "%s/%s", SCREAM_DATA_DIR, path); -} + // get column geometry and locations + col_latitudes_ = grid_->get_geometry_data("lat").get_view(); -} + // define the different field layouts that will be used for this process + using namespace ShortFieldTagsNames; -#define set_file_location(data_file, path) set_data_file(#data_file, path, data_file) + // Layout for 2D (2d horiz) variable + const FieldLayout scalar2d = grid_->get_2d_scalar_layout(); -void MAMMicrophysics::set_defaults_() { - config_.amicphys.do_cond = true; - config_.amicphys.do_rename = true; - config_.amicphys.do_newnuc = true; - config_.amicphys.do_coag = true; + // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and + // interfaces + const FieldLayout scalar3d_mid = grid_->get_3d_scalar_layout(true); + const FieldLayout scalar3d_int = grid_->get_3d_scalar_layout(false); - config_.amicphys.nucleation = {}; - config_.amicphys.nucleation.dens_so4a_host = 1770.0; - config_.amicphys.nucleation.mw_so4a_host = 115.0; - config_.amicphys.nucleation.newnuc_method_user_choice = 2; - config_.amicphys.nucleation.pbl_nuc_wang2008_user_choice = 1; - config_.amicphys.nucleation.adjust_factor_pbl_ratenucl = 1.0; - config_.amicphys.nucleation.accom_coef_h2so4 = 1.0; - config_.amicphys.nucleation.newnuc_adjust_factor_dnaitdt = 1.0; + // For U and V components of wind + const FieldLayout vector3d = grid_->get_3d_vector_layout(true, 2); - // these parameters guide the coupling between parameterizations - // NOTE: mam4xx was ported with these parameters fixed, so it's probably not - // NOTE: safe to change these without code modifications. - config_.amicphys.gaexch_h2so4_uptake_optaa = 2; - config_.amicphys.newnuc_h2so4_conc_optaa = 2; + using namespace ekat::units; + constexpr auto q_unit = kg / kg; // units of mass mixing ratios of tracers + constexpr auto n_unit = 1 / kg; // units of number mixing ratios of tracers - //=========================================================== - // default data file locations (relative to SCREAM_DATA_DIR) - //=========================================================== + // -------------------------------------------------------------------------- + // These variables are "Required" or pure inputs for the process + // -------------------------------------------------------------------------- - // many of these paths were extracted from - // e3smv2/bld/namelist_files/namelist_defaults_eam.xml + // ----------- Atmospheric quantities ------------- - // photolysis - set_file_location(config_.photolysis.rsf_file, "../waccm/phot/RSF_GT200nm_v3.0_c080811.nc"); - set_file_location(config_.photolysis.xs_long_file, "../waccm/phot/temp_prs_GT200nm_JPL10_c130206.nc"); + // Specific humidity [kg/kg](Require only for building DS) + add_tracer("qv", grid_, kg / kg); // specific humidity - // stratospheric chemistry - set_file_location(config_.linoz.chlorine_loading_file, "../cam/chem/trop_mozart/ub/Linoz_Chlorine_Loading_CMIP6_0003-2017_c20171114.nc"); + // Cloud liquid mass mixing ratio [kg/kg](Require only for building DS) + add_tracer("qc", grid_, kg / kg); // cloud liquid wet mixing ratio - // dry deposition - set_file_location(config_.drydep.srf_file, "../cam/chem/trop_mam/atmsrf_ne4pg2_200527.nc"); -} + // Cloud ice mass mixing ratio [kg/kg](Require only for building DS) + add_tracer("qi", grid_, kg / kg); // ice wet mixing ratio -void MAMMicrophysics::configure(const ekat::ParameterList& params) { - set_defaults_(); - // FIXME: implement "namelist" parsing -} + // Cloud liquid number mixing ratio [1/kg](Require only for building DS) + add_tracer("nc", grid_, + n_unit); // cloud liquid wet number mixing ratio -void MAMMicrophysics::set_grids(const std::shared_ptr grids_manager) { - using namespace ekat::units; + // Cloud ice number mixing ratio [1/kg](Require only for building DS) + add_tracer("ni", grid_, n_unit); // ice number mixing ratio - Units nondim = Units::nondimensional(); - Units n_unit (1/kg,"#/kg"); // number mixing ratios [# / kg air] - const auto m2 = pow(m,2); - const auto s2 = pow(s,2); + // Temperature[K] at midpoints + add_field("T_mid", scalar3d_mid, K, grid_name); - grid_ = grids_manager->get_grid("Physics"); - const auto& grid_name = grid_->name(); + // Vertical pressure velocity [Pa/s] at midpoints (Require only for building + // DS) + add_field("omega", scalar3d_mid, Pa / s, grid_name); - ncol_ = grid_->get_num_local_dofs(); // number of columns on this rank - nlev_ = grid_->get_num_vertical_levels(); // number of levels per column + // Total pressure [Pa] at midpoints + add_field("p_mid", scalar3d_mid, Pa, grid_name); - // get column geometry and locations - col_areas_ = grid_->get_geometry_data("area").get_view(); - col_latitudes_ = grid_->get_geometry_data("lat").get_view(); - col_longitudes_ = grid_->get_geometry_data("lon").get_view(); + // Total pressure [Pa] at interfaces + add_field("p_int", scalar3d_int, Pa, grid_name); - // define the different field layouts that will be used for this process - using namespace ShortFieldTagsNames; + // Layer thickness(pdel) [Pa] at midpoints + add_field("pseudo_density", scalar3d_mid, Pa, grid_name); - // layout for 2D (1d horiz X 1d vertical) variable - FieldLayout scalar2d_layout_col{ {COL}, {ncol_} }; - - // layout for 3D (2d horiz X 1d vertical) variables - FieldLayout scalar3d_layout_mid{ {COL, LEV}, {ncol_, nlev_} }; - - // define fields needed in mam4xx - - // atmospheric quantities - add_field("omega", scalar3d_layout_mid, Pa/s, grid_name); // vertical pressure velocity - add_field("T_mid", scalar3d_layout_mid, K, grid_name); // Temperature - add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); // total pressure - add_field("qv", scalar3d_layout_mid, kg/kg, grid_name, "tracers"); // specific humidity - add_field("qi", scalar3d_layout_mid, kg/kg, grid_name, "tracers"); // ice wet mixing ratio - add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers"); // ice number mixing ratio - add_field("pbl_height", scalar2d_layout_col, m, grid_name); // planetary boundary layer height - add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); // p_del, hydrostatic pressure - add_field("phis", scalar2d_layout_col, m2/s2, grid_name); - add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction - - // droplet activation can alter cloud liquid and number mixing ratios - add_field("qc", scalar3d_layout_mid, kg/kg, grid_name, "tracers"); // cloud liquid wet mixing ratio - add_field("nc", scalar3d_layout_mid, n_unit, grid_name, "tracers"); // cloud liquid wet number mixing ratio - - // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing ratios - for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); - add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); - for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { - const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); - if (strlen(int_mmr_field_name) > 0) { - add_field(int_mmr_field_name, scalar3d_layout_mid, kg/kg, grid_name, "tracers"); - } - } - } + // Planetary boundary layer height [m] + add_field("pbl_height", scalar2d, m, grid_name); - // aerosol-related gases: mass mixing ratios - for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - const char* gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - add_field(gas_mmr_field_name, scalar3d_layout_mid, kg/kg, grid_name, "tracers"); - } + constexpr auto m2 = pow(m, 2); + constexpr auto s2 = pow(s, 2); - // Tracers group -- do we need this in addition to the tracers above? In any - // case, this call should be idempotent, so it can't hurt. - add_group("tracers", grid_name, 1, Bundling::Required); -} + // Surface geopotential [m2/s2] + add_field("phis", scalar2d, m2 / s2, grid_name); -// this checks whether we have the tracers we expect -void MAMMicrophysics:: -set_computed_group_impl(const FieldGroup& group) { - const auto& name = group.m_info->m_group_name; - EKAT_REQUIRE_MSG(name=="tracers", - "Error! MAM4 expects a 'tracers' field group (got '" << name << "')\n"); - - EKAT_REQUIRE_MSG(group.m_info->m_bundled, - "Error! MAM4 expects bundled fields for tracers.\n"); - - // how many aerosol/gas tracers do we expect? - int num_tracers = 2 * (mam_coupling::num_aero_modes() + - mam_coupling::num_aero_tracers()) + - mam_coupling::num_aero_gases(); - EKAT_REQUIRE_MSG(group.m_info->size() >= num_tracers, - "Error! MAM4 requires at least " << num_tracers << " aerosol tracers."); -} + // Surface pressure [Pa] + add_field("ps", scalar2d, Pa, grid_name); + + // U and V components of the wind[m/s] + add_field("horiz_winds", vector3d, m / s, grid_name); + + //----------- Variables from microphysics scheme ------------- + constexpr auto nondim = ekat::units::Units::nondimensional(); + // Total cloud fraction [fraction] + add_field("cldfrac_liq", scalar3d_mid, nondim, grid_name); + + // Evaporation from stratiform rain [kg/kg/s] + add_field("nevapr", scalar3d_mid, kg / kg / s, grid_name); + + // Stratiform rain production rate [kg/kg/s] + add_field("precip_total_tend", scalar3d_mid, kg / kg / s, + grid_name); + + // precipitation liquid mass [kg/m2] + add_field("precip_liq_surf_mass", scalar3d_mid, kg / m2, grid_name); + + // precipitation ice mass [kg/m2] + add_field("precip_ice_surf_mass", scalar3d_mid, kg / m2, grid_name); + + //----------- Variables from other mam4xx processes ------------ + // Number of modes + constexpr int nmodes = mam4::AeroConfig::num_modes(); + + // layout for 3D (ncol, nmodes, nlevs) + FieldLayout scalar3d_mid_nmodes = + grid_->get_3d_vector_layout(true, nmodes, "nmodes"); + + // Geometric mean dry diameter for number distribution [m] + add_field("dgnum", scalar3d_mid_nmodes, m, grid_name); + // Geometric mean wet diameter for number distribution [m] + add_field("dgnumwet", scalar3d_mid_nmodes, m, grid_name); + + constexpr auto m3 = pow(m, 3); + // Wet density of interstitial aerosol [kg/m3] + add_field("wetdens", scalar3d_mid_nmodes, kg / m3, grid_name); + + // For fractional land use + const FieldLayout vector2d_class = + grid_->get_2d_vector_layout(mam4::mo_drydep::n_land_type, "class"); + + // Fractional land use [fraction] + add_field("fraction_landuse", vector2d_class, nondim, grid_name); + + //----------- Variables from the coupler --------- + // surface albedo shortwave, direct + add_field("sfc_alb_dir_vis", scalar2d, nondim, grid_name); + + // Surface temperature[K] + add_field("surf_radiative_T", scalar2d, K, grid_name); -size_t MAMMicrophysics::requested_buffer_size_in_bytes() const -{ + // snow depth land [m] + add_field("snow_depth_land", scalar2d, m, grid_name); + + //----------- Variables from the RRTMGP radiation --------- + // Downwelling solar flux at the surface [w/m2] + add_field("SW_flux_dn", scalar3d_int, W / m2, grid_name); + + // --------------------------------------------------------------------- + // These variables are "updated" or inputs/outputs for the process + // --------------------------------------------------------------------- + + // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing + // ratios + for(int m = 0; m < nmodes; ++m) { + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + + add_tracer(int_nmr_field_name, grid_, n_unit); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(m, a); + + if(strlen(int_mmr_field_name) > 0) { + add_tracer(int_mmr_field_name, grid_, kg / kg); + } + } // for loop species + } // for loop nmodes interstitial + // (cloud) aerosol tracers of interest: mass (q) and number (n) mixing ratios + for(int m = 0; m < nmodes; ++m) { + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + + add_field(cld_nmr_field_name, scalar3d_mid, n_unit, grid_name); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(m, a); + + if(strlen(cld_mmr_field_name) > 0) { + add_field(cld_mmr_field_name, scalar3d_mid, q_unit, grid_name); + } + } // for loop species + } // for loop nmodes cld borne + + // aerosol-related gases: mass mixing ratios + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + add_tracer(gas_mmr_field_name, grid_, kg / kg); + } + //----------- Updated variables from other mam4xx processes ------------ + // layout for Constituent fluxes + FieldLayout scalar2d_pcnst = + grid_->get_2d_vector_layout(mam4::pcnst, "num_phys_constituents"); + + // Constituent fluxes of species in [kg/m2/s] + add_field("constituent_fluxes", scalar2d_pcnst, kg / m2 / s, + grid_name); + + // Creating a Linoz reader and setting Linoz parameters involves reading data + // from a file and configuring the necessary parameters for the Linoz model. + { + linoz_file_name_ = m_params.get("mam4_linoz_file_name"); + const std::string linoz_map_file = + m_params.get("aero_microphys_remap_file", ""); + const std::vector var_names{ + "o3_clim", "o3col_clim", "t_clim", "PmL_clim", + "dPmL_dO3", "dPmL_dT", "dPmL_dO3col", "cariolle_pscs"}; + + // in format YYYYMMDD + const int linoz_cyclical_ymd = m_params.get("mam4_linoz_ymd"); + scream::mam_coupling::setup_tracer_data(linoz_data_, linoz_file_name_, + linoz_cyclical_ymd); + LinozHorizInterp_ = scream::mam_coupling::create_horiz_remapper( + grid_, linoz_file_name_, linoz_map_file, var_names, linoz_data_); + LinozDataReader_ = scream::mam_coupling::create_tracer_data_reader( + LinozHorizInterp_, linoz_file_name_); + + // linoz reader + const auto io_grid_linoz = LinozHorizInterp_->get_tgt_grid(); + const int num_cols_io_linoz = + io_grid_linoz->get_num_local_dofs(); // Number of columns on this rank + const int num_levs_io_linoz = + io_grid_linoz + ->get_num_vertical_levels(); // Number of levels per column + const int nvars = int(var_names.size()); + linoz_data_.init(num_cols_io_linoz, num_levs_io_linoz, nvars); + linoz_data_.allocate_temporal_views(); + } // LINOZ reader + + { + oxid_file_name_ = m_params.get("mam4_oxid_file_name"); + const std::string oxid_map_file = + m_params.get("aero_microphys_remap_file", ""); + // NOTE: order matches mam4xx: + const std::vector var_names{"O3", "OH", "NO3", "HO2"}; + + // in format YYYYMMDD + const int oxid_ymd = m_params.get("mam4_oxid_ymd"); + scream::mam_coupling::setup_tracer_data(tracer_data_, oxid_file_name_, + oxid_ymd); + TracerHorizInterp_ = scream::mam_coupling::create_horiz_remapper( + grid_, oxid_file_name_, oxid_map_file, var_names, tracer_data_); + TracerDataReader_ = scream::mam_coupling::create_tracer_data_reader( + TracerHorizInterp_, oxid_file_name_); + + const int nvars = int(var_names.size()); + const auto io_grid = TracerHorizInterp_->get_tgt_grid(); + const int num_cols_io = + io_grid->get_num_local_dofs(); // Number of columns on this rank + const int num_levs_io = + io_grid->get_num_vertical_levels(); // Number of levels per column + tracer_data_.init(num_cols_io, num_levs_io, nvars); + tracer_data_.allocate_temporal_views(); + + for(int ivar = 0; ivar < nvars; ++ivar) { + cnst_offline_[ivar] = view_2d("cnst_offline_", ncol_, nlev_); + } + } // oxid file reader + + { + const std::string extfrc_map_file = + m_params.get("aero_microphys_remap_file", ""); + // NOTE: order of forcing species is important. + // extfrc_lst(: 9) = {'SO2 ','so4_a1 ','so4_a2 + // ','pom_a4 ','bc_a4 ', 'num_a1 ','num_a2 + // ','num_a4 ','SOAG ' } + // This order corresponds to files in namelist e3smv2 + extfrc_lst_ = {"so2", "so4_a1", "so4_a2", "pom_a4", "bc_a4", + "num_a1", "num_a2", "num_a4", "soag"}; + + for(const auto &var_name : extfrc_lst_) { + std::string item_name = "mam4_" + var_name + "_elevated_emiss_file_name"; + const auto file_name = m_params.get(item_name); + elevated_emis_file_name_[var_name] = file_name; + } + elevated_emis_var_names_["so2"] = {"BB", "ENE_ELEV", "IND_ELEV", + "contvolc"}; + elevated_emis_var_names_["so4_a1"] = {"BB", "ENE_ELEV", "IND_ELEV", + "contvolc"}; + elevated_emis_var_names_["so4_a2"] = {"contvolc"}; + elevated_emis_var_names_["pom_a4"] = {"BB"}; + elevated_emis_var_names_["bc_a4"] = {"BB"}; + elevated_emis_var_names_["num_a1"] = { + "num_a1_SO4_ELEV_BB", "num_a1_SO4_ELEV_ENE", "num_a1_SO4_ELEV_IND", + "num_a1_SO4_ELEV_contvolc"}; + elevated_emis_var_names_["num_a2"] = {"num_a2_SO4_ELEV_contvolc"}; + // num_a4 + // FIXME: why the sectors in this files are num_a1; + // I guess this should be num_a4? Is this a bug in the orginal nc files? + elevated_emis_var_names_["num_a4"] = {"num_a1_BC_ELEV_BB", + "num_a1_POM_ELEV_BB"}; + elevated_emis_var_names_["soag"] = {"SOAbb_src", "SOAbg_src", "SOAff_src"}; + + int elevated_emiss_cyclical_ymd = m_params.get("elevated_emiss_ymd"); + + for(const auto &var_name : extfrc_lst_) { + const auto file_name = elevated_emis_file_name_[var_name]; + const auto var_names = elevated_emis_var_names_[var_name]; + + scream::mam_coupling::TracerData data_tracer; + scream::mam_coupling::setup_tracer_data(data_tracer, file_name, + elevated_emiss_cyclical_ymd); + auto hor_rem = scream::mam_coupling::create_horiz_remapper( + grid_, file_name, extfrc_map_file, var_names, data_tracer); + + auto file_reader = scream::mam_coupling::create_tracer_data_reader( + hor_rem, file_name, data_tracer.file_type); + ElevatedEmissionsHorizInterp_.push_back(hor_rem); + ElevatedEmissionsDataReader_.push_back(file_reader); + elevated_emis_data_.push_back(data_tracer); + } // var_name elevated emissions + int i = 0; + int offset_emis_ver = 0; + for(const auto &var_name : extfrc_lst_) { + const auto file_name = elevated_emis_file_name_[var_name]; + const auto var_names = elevated_emis_var_names_[var_name]; + const int nvars = static_cast(var_names.size()); + + forcings_[i].nsectors = nvars; + // I am assuming the order of species in extfrc_lst_. + // Indexing in mam4xx is fortran. + forcings_[i].frc_ndx = i + 1; + const auto io_grid_emis = + ElevatedEmissionsHorizInterp_[i]->get_tgt_grid(); + const int num_cols_io_emis = + io_grid_emis->get_num_local_dofs(); // Number of columns on this rank + const int num_levs_io_emis = + io_grid_emis + ->get_num_vertical_levels(); // Number of levels per column + elevated_emis_data_[i].init(num_cols_io_emis, num_levs_io_emis, nvars); + elevated_emis_data_[i].allocate_temporal_views(); + forcings_[i].file_alt_data = elevated_emis_data_[i].has_altitude_; + for(int isp = 0; isp < nvars; ++isp) { + forcings_[i].offset = offset_emis_ver; + elevated_emis_output_[isp + offset_emis_ver] = + view_2d("elevated_emis_output_", ncol_, nlev_); + } + offset_emis_ver += nvars; + ++i; + } // end i + EKAT_REQUIRE_MSG( + offset_emis_ver <= int(mam_coupling::MAX_NUM_ELEVATED_EMISSIONS_FIELDS), + "Error! Number of fields is bigger than " + "MAX_NUM_ELEVATED_EMISSIONS_FIELDS. Increase the " + "MAX_NUM_ELEVATED_EMISSIONS_FIELDS in tracer_reader_utils.hpp \n"); + + } // Tracer external forcing data + + { + const std::string season_wes_file = + m_params.get("mam4_season_wes_file"); + const auto &clat = col_latitudes_; + mam_coupling::find_season_index_reader(season_wes_file, clat, + index_season_lai_); + } +} // set_grids + +// ================================================================ +// REQUEST_BUFFER_SIZE_IN_BYTES +// ================================================================ +// ON HOST, returns the number of bytes of device memory needed by +// the above. Buffer type given the number of columns and vertical +// levels +size_t MAMMicrophysics::requested_buffer_size_in_bytes() const { return mam_coupling::buffer_size(ncol_, nlev_); } -void MAMMicrophysics::init_buffers(const ATMBufferManager &buffer_manager) { - EKAT_REQUIRE_MSG(buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), - "Error! Insufficient buffer size.\n"); +// ================================================================ +// INIT_BUFFERS +// ================================================================ +// ON HOST, initializeѕ the Buffer type with sufficient memory to +// store intermediate (dry) quantities on the given number of +// columns with the given number of vertical levels. Returns the +// number of bytes allocated. - size_t used_mem = mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); - EKAT_REQUIRE_MSG(used_mem==requested_buffer_size_in_bytes(), - "Error! Used memory != requested memory for MAMMicrophysics."); +void MAMMicrophysics::init_buffers(const ATMBufferManager &buffer_manager) { + size_t used_mem = + mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); + EKAT_REQUIRE_MSG(used_mem == requested_buffer_size_in_bytes(), + "Error! Used memory != requested memory for MAMMicrophysics." + " Used memory: " + << std::to_string(used_mem) + << "." + " Requested memory: " + << std::to_string(requested_buffer_size_in_bytes()) + << ". \n"); } - +// ================================================================ +// INITIALIZE_IMPL +// ================================================================ void MAMMicrophysics::initialize_impl(const RunType run_type) { - - step_ = 0; - - // populate the wet and dry atmosphere states with views from fields and - // the buffer - wet_atm_.qv = get_field_in("qv").get_view(); - wet_atm_.qc = get_field_out("qc").get_view(); - wet_atm_.nc = get_field_out("nc").get_view(); - wet_atm_.qi = get_field_in("qi").get_view(); - wet_atm_.ni = get_field_in("ni").get_view(); - - - dry_atm_.T_mid = get_field_in("T_mid").get_view(); - dry_atm_.p_mid = get_field_in("p_mid").get_view(); - dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); // FIXME: tot or liq? - dry_atm_.pblh = get_field_in("pbl_height").get_view(); - dry_atm_.phis = get_field_in("phis").get_view(); - dry_atm_.omega = get_field_in("omega").get_view(); + // Determine orbital year. If orbital_year is negative, use current year + // from timestamp for orbital year; if positive, use provided orbital year + // for duration of simulation. + m_orbital_year = m_params.get("orbital_year", -9999); + + // Get orbital parameters from yaml file + m_orbital_eccen = m_params.get("orbital_eccentricity", -9999); + m_orbital_obliq = m_params.get("orbital_obliquity", -9999); + m_orbital_mvelp = m_params.get("orbital_mvelp", -9999); + + // --------------------------------------------------------------- + // Input fields read in from IC file, namelist or other processes + // --------------------------------------------------------------- + + // Populate the wet atmosphere state with views from fields + // FIMXE: specifically look which among these are actually used by the process + + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_int = get_field_in("p_int").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.cldfrac = get_field_in("cldfrac_liq").get_view(); + dry_atm_.pblh = get_field_in("pbl_height").get_view(); + dry_atm_.phis = get_field_in("phis").get_view(); + dry_atm_.omega = get_field_in("omega").get_view(); dry_atm_.z_mid = buffer_.z_mid; dry_atm_.dz = buffer_.dz; dry_atm_.z_iface = buffer_.z_iface; @@ -219,43 +468,60 @@ void MAMMicrophysics::initialize_impl(const RunType run_type) { dry_atm_.qi = buffer_.qi_dry; dry_atm_.ni = buffer_.ni_dry; dry_atm_.w_updraft = buffer_.w_updraft; - dry_atm_.z_surf = 0.0; // FIXME: for now - - // perform any initialization work - if (run_type==RunType::Initial) { - } - - // set wet/dry aerosol state data (interstitial aerosols only) - for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); - wet_aero_.int_aero_nmr[m] = get_field_out(int_nmr_field_name).get_view(); + dry_atm_.z_surf = 0.0; // It is always zero. + + // get surface albedo: shortwave, direct + d_sfc_alb_dir_vis_ = get_field_in("sfc_alb_dir_vis").get_view(); + + // interstitial and cloudborne aerosol tracers of interest: mass (q) and + // number (n) mixing ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + // interstitial aerosol tracers of interest: number (n) mixing ratios + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + wet_aero_.int_aero_nmr[m] = + get_field_out(int_nmr_field_name).get_view(); dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; - for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { - const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); - if (strlen(int_mmr_field_name) > 0) { - wet_aero_.int_aero_mmr[m][a] = get_field_out(int_mmr_field_name).get_view(); + + // cloudborne aerosol tracers of interest: number (n) mixing ratios + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + wet_aero_.cld_aero_nmr[m] = + get_field_out(cld_nmr_field_name).get_view(); + dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; + + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + // (interstitial) aerosol tracers of interest: mass (q) mixing ratios + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(m, a); + if(strlen(int_mmr_field_name) > 0) { + wet_aero_.int_aero_mmr[m][a] = + get_field_out(int_mmr_field_name).get_view(); dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; } - } - } + + // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(m, a); + if(strlen(cld_mmr_field_name) > 0) { + wet_aero_.cld_aero_mmr[m][a] = + get_field_out(cld_mmr_field_name).get_view(); + dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; + } + } // for loop species + } // for loop num_aero_modes() // set wet/dry aerosol-related gas state data - for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - const char* mmr_field_name = mam_coupling::gas_mmr_field_name(g); - wet_aero_.gas_mmr[g] = get_field_out(mmr_field_name).get_view(); + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *mmr_field_name = mam_coupling::gas_mmr_field_name(g); + wet_aero_.gas_mmr[g] = get_field_out(mmr_field_name).get_view(); dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; } // create our photolysis rate calculation table - photo_table_ = impl::read_photo_table(get_comm(), - config_.photolysis.rsf_file, - config_.photolysis.xs_long_file); - - // FIXME: read relevant land use data from drydep surface file + const std::string rsf_file = m_params.get("mam4_rsf_file"); + const std::string xs_long_file = + m_params.get("mam4_xs_long_file"); - // set up our preprocess/postprocess functors - preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); - postprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); + photo_table_ = impl::read_photo_table(rsf_file, xs_long_file); // set field property checks for the fields in this process /* e.g. @@ -267,249 +533,467 @@ void MAMMicrophysics::initialize_impl(const RunType run_type) { add_postcondition_check(get_field_out("tke"),m_grid,0); */ - // set up WSM for internal local variables - // (we'll probably need this later, but we'll just use ATMBufferManager for now) - //const auto default_policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); - //workspace_mgr_.setup(buffer_.wsm_data, nlev_+1, 13+(n_wind_slots+n_trac_slots), default_policy); -} + { + // climatology data for linear stratospheric chemistry + auto linoz_o3_clim = buffer_.scratch[0]; // ozone (climatology) [vmr] + auto linoz_o3col_clim = + buffer_.scratch[1]; // column o3 above box (climatology) [Dobson Units + // (DU)] + auto linoz_t_clim = buffer_.scratch[2]; // temperature (climatology) [K] + auto linoz_PmL_clim = + buffer_.scratch[3]; // P minus L (climatology) [vmr/s] + auto linoz_dPmL_dO3 = + buffer_.scratch[4]; // sensitivity of P minus L to O3 [1/s] + auto linoz_dPmL_dT = + buffer_.scratch[5]; // sensitivity of P minus L to T3 [K] + auto linoz_dPmL_dO3col = buffer_.scratch[6]; // sensitivity of P minus L to + // overhead O3 column [vmr/DU] + auto linoz_cariolle_pscs = + buffer_.scratch[7]; // Cariolle parameter for PSC loss of ozone [1/s] + + auto ts = timestamp(); + std::string linoz_chlorine_file = + m_params.get("mam4_linoz_chlorine_file"); + int chlorine_loading_ymd = m_params.get("mam4_chlorine_loading_ymd"); + scream::mam_coupling::create_linoz_chlorine_reader( + linoz_chlorine_file, ts, chlorine_loading_ymd, chlorine_values_, + chlorine_time_secs_); + } // LINOZ + + const int photo_table_len = get_photo_table_work_len(photo_table_); + work_photo_table_ = view_2d("work_photo_table", ncol_, photo_table_len); + const int sethet_work_len = mam4::mo_sethet::get_total_work_len_sethet(); + work_set_het_ = view_2d("work_set_het_array", ncol_, sethet_work_len); + cmfdqr_ = view_1d("cmfdqr_", nlev_); -void MAMMicrophysics::run_impl(const double dt) { + // here's where we store per-column photolysis rates + photo_rates_ = view_3d("photo_rates", ncol_, nlev_, mam4::mo_photo::phtcnt); - const auto scan_policy = ekat::ExeSpaceUtils::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); - const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); + // Load the first month into extfrc_lst_end. + // Note: At the first time step, the data will be moved into extfrc_lst_beg, + // and extfrc_lst_end will be reloaded from file with the new month. + const int curr_month = timestamp().get_month() - 1; // 0-based + + scream::mam_coupling::update_tracer_data_from_file( + LinozDataReader_, curr_month, *LinozHorizInterp_, linoz_data_); + + scream::mam_coupling::update_tracer_data_from_file( + TracerDataReader_, curr_month, *TracerHorizInterp_, tracer_data_); + + for(int i = 0; i < static_cast(extfrc_lst_.size()); ++i) { + scream::mam_coupling::update_tracer_data_from_file( + ElevatedEmissionsDataReader_[i], curr_month, + *ElevatedEmissionsHorizInterp_[i], elevated_emis_data_[i]); + } + + invariants_ = view_3d("invarians", ncol_, nlev_, mam4::gas_chemistry::nfs); + + constexpr int extcnt = mam4::gas_chemistry::extcnt; + extfrc_ = view_3d("extfrc_", ncol_, nlev_, extcnt); + + // + acos_cosine_zenith_host_ = view_1d_host("host_acos(cosine_zenith)", ncol_); + acos_cosine_zenith_ = view_1d("device_acos(cosine_zenith)", ncol_); + + //----------------------------------------------------------------- + // Setup preprocessing and post processing + //----------------------------------------------------------------- + preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, + dry_aero_); + postprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, + dry_aero_); + +} // initialize_impl + +// ================================================================ +// RUN_IMPL +// ================================================================ +void MAMMicrophysics::run_impl(const double dt) { + const int ncol = ncol_; + const int nlev = nlev_; + const auto scan_policy = ekat::ExeSpaceUtils< + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol, nlev); + const auto policy = + ekat::ExeSpaceUtils::get_default_team_policy(ncol, nlev); // preprocess input -- needs a scan for the calculation of atm height Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - // reset internal WSM variables - //workspace_mgr_.reset_internals(); + //----------- Variables from microphysics scheme ------------- - // NOTE: nothing depends on simulation time (yet), so we can just use zero for now - double t = 0.0; + // Evaporation from stratiform rain [kg/kg/s] + const auto &nevapr = get_field_in("nevapr").get_view(); - // here's where we store per-column photolysis rates - using View2D = haero::DeviceType::view_2d; - View2D photo_rates("photo_rates", nlev_, mam4::mo_photo::phtcnt); + // Stratiform rain production rate [kg/kg/s] + const auto &prain = + get_field_in("precip_total_tend").get_view(); - // climatology data for linear stratospheric chemistry - auto linoz_o3_clim = buffer_.scratch[0]; // ozone (climatology) [vmr] - auto linoz_o3col_clim = buffer_.scratch[1]; // column o3 above box (climatology) [Dobson Units (DU)] - auto linoz_t_clim = buffer_.scratch[2]; // temperature (climatology) [K] - auto linoz_PmL_clim = buffer_.scratch[3]; // P minus L (climatology) [vmr/s] - auto linoz_dPmL_dO3 = buffer_.scratch[4]; // sensitivity of P minus L to O3 [1/s] - auto linoz_dPmL_dT = buffer_.scratch[5]; // sensitivity of P minus L to T3 [K] - auto linoz_dPmL_dO3col = buffer_.scratch[6]; // sensitivity of P minus L to overhead O3 column [vmr/DU] - auto linoz_cariolle_psc = buffer_.scratch[7]; // Cariolle parameter for PSC loss of ozone [1/s] + const auto wet_geometric_mean_diameter_i = + get_field_in("dgnumwet").get_view(); + const auto dry_geometric_mean_diameter_i = + get_field_in("dgnum").get_view(); + const auto wetdens = get_field_in("wetdens").get_view(); + + // U wind component [m/s] + const const_view_2d u_wind = + get_field_in("horiz_winds").get_component(0).get_view(); + + // V wind component [m/s] + const const_view_2d v_wind = + get_field_in("horiz_winds").get_component(1).get_view(); + // Liquid precip [kg/m2] + const const_view_2d precip_liq_surf_mass = + get_field_in("precip_liq_surf_mass").get_view(); + + // Ice precip [kg/m2] + const const_view_2d precip_ice_surf_mass = + get_field_in("precip_ice_surf_mass").get_view(); + + // Fractional land use [fraction] + const const_view_2d fraction_landuse = + get_field_in("fraction_landuse").get_view(); + + // Downwelling solar flux at the surface [w/m2] + const const_view_2d sw_flux_dn = + get_field_in("SW_flux_dn").get_view(); + + // Constituent fluxes of gas and aerosol species + view_2d constituent_fluxes = + get_field_out("constituent_fluxes").get_view(); + + // Surface temperature [K] + const const_view_1d sfc_temperature = + get_field_in("surf_radiative_T").get_view(); + + // Surface pressure [Pa] + const const_view_1d sfc_pressure = + get_field_in("ps").get_view(); + + // Snow depth on land [m] + const const_view_1d snow_depth_land = + get_field_in("snow_depth_land").get_view(); + + // climatology data for linear stratospheric chemistry + // ozone (climatology) [vmr] + auto linoz_o3_clim = buffer_.scratch[0]; + // column o3 above box (climatology) [Dobson Units (DU)] + auto linoz_o3col_clim = buffer_.scratch[1]; + auto linoz_t_clim = buffer_.scratch[2]; // temperature (climatology) [K] + auto linoz_PmL_clim = buffer_.scratch[3]; // P minus L (climatology) [vmr/s] + // sensitivity of P minus L to O3 [1/s] + auto linoz_dPmL_dO3 = buffer_.scratch[4]; + // sensitivity of P minus L to T3 [K] + auto linoz_dPmL_dT = buffer_.scratch[5]; + // sensitivity of P minus L to overhead O3 column [vmr/DU] + auto linoz_dPmL_dO3col = buffer_.scratch[6]; + // Cariolle parameter for PSC loss of ozone [1/s] + auto linoz_cariolle_pscs = buffer_.scratch[7]; + + view_2d linoz_output[8]; + linoz_output[0] = linoz_o3_clim; + linoz_output[1] = linoz_o3col_clim; + linoz_output[2] = linoz_t_clim; + linoz_output[3] = linoz_PmL_clim; + linoz_output[4] = linoz_dPmL_dO3; + linoz_output[5] = linoz_dPmL_dT; + linoz_output[6] = linoz_dPmL_dO3col; + linoz_output[7] = linoz_cariolle_pscs; // it's a bit wasteful to store this for all columns, but simpler from an // allocation perspective auto o3_col_dens = buffer_.scratch[8]; - const_view_1d &col_latitudes = col_latitudes_; - mam_coupling::DryAtmosphere &dry_atm = dry_atm_; - mam_coupling::AerosolState &dry_aero = dry_aero_; - mam4::mo_photo::PhotoTableData &photo_table = photo_table_; - const int nlev = nlev_; - const Config &config = config_; - // FIXME: read relevant linoz climatology data from file(s) based on time - - // FIXME: read relevant chlorine loading data from file based on time + /* Gather time and state information for interpolation */ + const auto ts = timestamp() + dt; + + const Real chlorine_loading = scream::mam_coupling::chlorine_loading_advance( + ts, chlorine_values_, chlorine_time_secs_); + + // /* Update the TracerTimeState to reflect the current time, note the + // addition of dt */ + trace_time_state_.t_now = ts.frac_of_year_in_days(); + scream::mam_coupling::advance_tracer_data( + TracerDataReader_, // in + *TracerHorizInterp_, // out + ts, // in + trace_time_state_, tracer_data_, // out + dry_atm_.p_mid, dry_atm_.z_iface, // in + cnst_offline_); // out + Kokkos::fence(); - // loop over atmosphere columns and compute aerosol microphyscs - auto some_step = step_; - - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const ThreadTeam& team) { - const int icol = team.league_rank(); // column index - - Real col_lat = col_latitudes(icol); // column latitude (degrees?) - - // fetch column-specific atmosphere state data - auto atm = mam_coupling::atmosphere_for_column(dry_atm, icol); - auto z_iface = ekat::subview(dry_atm.z_iface, icol); - Real phis = dry_atm.phis(icol); - - // set surface state data - haero::Surface sfc{}; - - // fetch column-specific subviews into aerosol prognostics - mam4::Prognostics progs = mam_coupling::interstitial_aerosols_for_column(dry_aero, icol); - - // set up diagnostics - mam4::Diagnostics diags(nlev); - - // calculate o3 column densities (first component of col_dens in Fortran code) - auto o3_col_dens_i = ekat::subview(o3_col_dens, icol); - impl::compute_o3_column_density(team, atm, progs, o3_col_dens_i); - - // set up photolysis work arrays for this column. - mam4::mo_photo::PhotoTableWorkArrays photo_work_arrays; - // FIXME: set views here - - // ... look up photolysis rates from our table - // NOTE: the table interpolation operates on an entire column of data, so we - // NOTE: must do it before dispatching to individual vertical levels - Real zenith_angle = 0.0; // FIXME: need to get this from EAMxx [radians] - Real surf_albedo = 0.0; // FIXME: surface albedo - Real esfact = 0.0; // FIXME: earth-sun distance factor - mam4::ColumnView lwc; // FIXME: liquid water cloud content: where do we get this? - mam4::mo_photo::table_photo(photo_rates, atm.pressure, atm.hydrostatic_dp, - atm.temperature, o3_col_dens_i, zenith_angle, surf_albedo, lwc, - atm.cloud_fraction, esfact, photo_table, photo_work_arrays); - - // compute external forcings at time t(n+1) [molecules/cm^3/s] - constexpr int extcnt = mam4::gas_chemistry::extcnt; - view_2d extfrc; // FIXME: where to allocate? (nlev, extcnt) - mam4::mo_setext::Forcing forcings[extcnt]; // FIXME: forcings seem to require file data - mam4::mo_setext::extfrc_set(forcings, extfrc); - - // compute aerosol microphysics on each vertical level within this column - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nlev), [&](const int k) { - - constexpr int num_modes = mam4::AeroConfig::num_modes(); - constexpr int gas_pcnst = mam_coupling::gas_pcnst(); - constexpr int nqtendbb = mam_coupling::nqtendbb(); - - // extract atm state variables (input) - Real temp = atm.temperature(k); - Real pmid = atm.pressure(k); - Real pdel = atm.hydrostatic_dp(k); - Real zm = atm.height(k); - Real zi = z_iface(k); - Real pblh = atm.planetary_boundary_layer_height; - Real qv = atm.vapor_mixing_ratio(k); - Real cldfrac = atm.cloud_fraction(k); - - // extract aerosol state variables into "working arrays" (mass mixing ratios) - // (in EAM, this is done in the gas_phase_chemdr subroutine defined within - // mozart/mo_gas_phase_chemdr.F90) - Real q[gas_pcnst] = {}; - Real qqcw[gas_pcnst] = {}; - mam_coupling::transfer_prognostics_to_work_arrays(progs, k, q, qqcw); - - // convert mass mixing ratios to volume mixing ratios (VMR), equivalent - // to tracer mixing ratios (TMR)) - Real vmr[gas_pcnst], vmrcw[gas_pcnst]; - mam_coupling::convert_work_arrays_to_vmr(q, qqcw, vmr, vmrcw); - - // aerosol/gas species tendencies (output) - Real vmr_tendbb[gas_pcnst][nqtendbb] = {}; - Real vmrcw_tendbb[gas_pcnst][nqtendbb] = {}; - - // create work array copies to retain "pre-chemistry" values - Real vmr_pregaschem[gas_pcnst] = {}; - Real vmr_precldchem[gas_pcnst] = {}; - Real vmrcw_precldchem[gas_pcnst] = {}; - for (int i = 0; i < gas_pcnst; ++i) { - vmr_pregaschem[i] = vmr[i]; - vmr_precldchem[i] = vmr[i]; - vmrcw_precldchem[i] = vmrcw[i]; - } + scream::mam_coupling::advance_tracer_data( + LinozDataReader_, // in + *LinozHorizInterp_, // out + ts, // in + linoz_time_state_, linoz_data_, // out + dry_atm_.p_mid, dry_atm_.z_iface, // in + linoz_output); // out + Kokkos::fence(); - //--------------------- - // Gas Phase Chemistry - //--------------------- - Real photo_rates_k[mam4::mo_photo::phtcnt]; - for (int i = 0; i < mam4::mo_photo::phtcnt; ++i) { - photo_rates_k[i] = photo_rates(k, i); - } - Real extfrc_k[extcnt]; - for (int i = 0; i < extcnt; ++i) { - extfrc_k[i] = extfrc(k, i); - } - constexpr int nfs = mam4::gas_chemistry::nfs; // number of "fixed species" - // NOTE: we compute invariants here and pass them out to use later with - // NOTE: setsox - Real invariants[nfs]; - impl::gas_phase_chemistry(zm, zi, phis, temp, pmid, pdel, dt, - photo_rates_k, extfrc_k, vmr, invariants); - - //---------------------- - // Aerosol microphysics - //---------------------- - // the logic below is taken from the aero_model_gasaerexch subroutine in - // eam/src/chemistry/modal_aero/aero_model.F90 - - // aqueous chemistry ... - const int loffset = 8; // offset of first tracer in work arrays - // (taken from mam4xx setsox validation test) - const Real mbar = haero::Constants::molec_weight_dry_air; - constexpr int indexm = 0; // FIXME: index of xhnm in invariants array (??) - Real cldnum = 0.0; // FIXME: droplet number concentration: where do we get this? - setsox_single_level(loffset, dt, pmid, pdel, temp, mbar, lwc(k), - cldfrac, cldnum, invariants[indexm], config.setsox, vmrcw, vmr); - - // calculate aerosol water content using water uptake treatment - // * dry and wet diameters [m] - // * wet densities [kg/m3] - // * aerosol water mass mixing ratio [kg/kg] - Real dgncur_a[num_modes] = {}; - Real dgncur_awet[num_modes] = {}; - Real wetdens[num_modes] = {}; - Real qaerwat[num_modes] = {}; - impl::compute_water_content(progs, k, qv, temp, pmid, dgncur_a, dgncur_awet, wetdens, qaerwat); - - // do aerosol microphysics (gas-aerosol exchange, nucleation, coagulation) - impl::modal_aero_amicphys_intr(config.amicphys, step_, dt, t, pmid, pdel, - zm, pblh, qv, cldfrac, vmr, vmrcw, vmr_pregaschem, - vmr_precldchem, vmrcw_precldchem, vmr_tendbb, - vmrcw_tendbb, dgncur_a, dgncur_awet, - wetdens, qaerwat); - - //----------------- - // LINOZ chemistry - //----------------- - - // the following things are diagnostics, which we're not - // including in the first rev - Real do3_linoz, do3_linoz_psc, ss_o3, o3col_du_diag, o3clim_linoz_diag, - zenith_angle_degrees; - - // FIXME: Need to get chlorine loading data from file - Real chlorine_loading = 0.0; - - Real rlats = col_lat * M_PI / 180.0; // convert column latitude to radians - int o3_ndx = 0; // index of "O3" in solsym array (in EAM) - mam4::lin_strat_chem::lin_strat_chem_solve_kk(o3_col_dens_i(k), temp, - zenith_angle, pmid, dt, rlats, - linoz_o3_clim(icol, k), linoz_t_clim(icol, k), linoz_o3col_clim(icol, k), - linoz_PmL_clim(icol, k), linoz_dPmL_dO3(icol, k), linoz_dPmL_dT(icol, k), - linoz_dPmL_dO3col(icol, k), linoz_cariolle_psc(icol, k), - chlorine_loading, config.linoz.psc_T, vmr[o3_ndx], - do3_linoz, do3_linoz_psc, ss_o3, - o3col_du_diag, o3clim_linoz_diag, zenith_angle_degrees); - - // update source terms above the ozone decay threshold - if (k > nlev - config.linoz.o3_lbl - 1) { - Real do3_mass; // diagnostic, not needed - mam4::lin_strat_chem::lin_strat_sfcsink_kk(dt, pdel, vmr[o3_ndx], config.linoz.o3_sfc, - config.linoz.o3_tau, do3_mass); - } + elevated_emiss_time_state_.t_now = ts.frac_of_year_in_days(); + int i = 0; + for(const auto &var_name : extfrc_lst_) { + const auto file_name = elevated_emis_file_name_[var_name]; + const auto var_names = elevated_emis_var_names_[var_name]; + const int nsectors = int(var_names.size()); + view_2d elevated_emis_output[nsectors]; + for(int isp = 0; isp < nsectors; ++isp) { + elevated_emis_output[isp] = + elevated_emis_output_[isp + forcings_[i].offset]; + } + scream::mam_coupling::advance_tracer_data( + ElevatedEmissionsDataReader_[i], *ElevatedEmissionsHorizInterp_[i], ts, + elevated_emiss_time_state_, elevated_emis_data_[i], dry_atm_.p_mid, + dry_atm_.z_iface, elevated_emis_output); + i++; + Kokkos::fence(); + } - // ... check for negative values and reset to zero - for (int i = 0; i < gas_pcnst; ++i) { - if (vmr[i] < 0.0) vmr[i] = 0.0; - } + const_view_1d &col_latitudes = col_latitudes_; + const_view_1d &d_sfc_alb_dir_vis = d_sfc_alb_dir_vis_; - //---------------------- - // Dry deposition (gas) - //---------------------- + mam_coupling::DryAtmosphere &dry_atm = dry_atm_; + mam_coupling::AerosolState &dry_aero = dry_aero_; - // FIXME: C++ port in progress! - //mam4::drydep::drydep_xactive(...); + mam4::mo_photo::PhotoTableData &photo_table = photo_table_; + const Config &config = config_; + const auto &work_photo_table = work_photo_table_; + const auto &photo_rates = photo_rates_; + + const auto &invariants = invariants_; + const auto &cnst_offline = cnst_offline_; + + // Compute orbital parameters; these are used both for computing + // the solar zenith angle. + // Note: We are following the RRTMGP EAMxx interface to compute the zenith + // angle. This operation is performed on the host because the routine + // shr_orb_cosz_c2f has not been ported to C++. + auto ts2 = timestamp(); + auto orbital_year = m_orbital_year; + // Note: We need double precision because + // shr_orb_params_c2f and shr_orb_decl_c2f only support double precision. + double obliqr, lambm0, mvelpp; + double eccen = m_orbital_eccen; + double obliq = m_orbital_obliq; + double mvelp = m_orbital_mvelp; + // Use the orbital parameters to calculate the solar declination and + // eccentricity factor + double delta, eccf; + if(eccen >= 0 && obliq >= 0 && mvelp >= 0) { + // use fixed orbital parameters; to force this, we need to set + // orbital_year to SHR_ORB_UNDEF_INT, which is exposed through + // our c2f bridge as shr_orb_undef_int_c2f + orbital_year = shr_orb_undef_int_c2f; + } else if(orbital_year < 0) { + // compute orbital parameters based on current year + orbital_year = ts2.get_year(); + } + shr_orb_params_c2f(&orbital_year, // in + &eccen, &obliq, &mvelp, &obliqr, &lambm0, &mvelpp); // out + + // Want day + fraction; calday 1 == Jan 1 0Z + auto calday = ts2.frac_of_year_in_days() + 1; + shr_orb_decl_c2f(calday, eccen, mvelpp, lambm0, obliqr, // in + &delta, &eccf); // out + { + const auto col_latitudes_host = + grid_->get_geometry_data("lat").get_view(); + const auto col_longitudes_host = + grid_->get_geometry_data("lon").get_view(); + // get a host copy of lat/lon + // Determine the cosine zenith angle + // NOTE: Since we are bridging to F90 arrays this must be done on HOST and + // then deep copied to a device view. + + // Now use solar declination to calculate zenith angle for all points + for(int i = 0; i < ncol; i++) { + Real lat = + col_latitudes_host(i) * M_PI / 180.0; // Convert lat/lon to radians + Real lon = col_longitudes_host(i) * M_PI / 180.0; + // what's the aerosol microphys frequency? + Real temp = shr_orb_cosz_c2f(calday, lat, lon, delta, dt); + acos_cosine_zenith_host_(i) = acos(temp); + } + Kokkos::deep_copy(acos_cosine_zenith_, acos_cosine_zenith_host_); + } + const auto zenith_angle = acos_cosine_zenith_; + constexpr int gas_pcnst = mam_coupling::gas_pcnst(); + + const auto &elevated_emis_output = elevated_emis_output_; + const auto &extfrc = extfrc_; + const auto &forcings = forcings_; + constexpr int extcnt = mam4::gas_chemistry::extcnt; + + const int offset_aerosol = mam4::utils::gasses_start_ind(); + Real adv_mass_kg_per_moles[gas_pcnst]; + // NOTE: Making copies of clsmap_4 and permute_4 to fix undefined arrays on + // the device. + int clsmap_4[gas_pcnst], permute_4[gas_pcnst]; + for(int i = 0; i < gas_pcnst; ++i) { + // NOTE: state_q is kg/kg-dry-air; adv_mass is in g/mole. + // Convert adv_mass to kg/mole as vmr_from_mmr function uses + // molec_weight_dry_air with kg/mole units + adv_mass_kg_per_moles[i] = mam4::gas_chemistry::adv_mass[i] / 1e3; + clsmap_4[i] = mam4::gas_chemistry::clsmap_4[i]; + permute_4[i] = mam4::gas_chemistry::permute_4[i]; + } + const auto &cmfdqr = cmfdqr_; + const auto &work_set_het = work_set_het_; + const mam4::seq_drydep::Data drydep_data = + mam4::seq_drydep::set_gas_drydep_data(); + const auto qv = wet_atm_.qv; + const int month = timestamp().get_month(); // 1-based + const int surface_lev = nlev - 1; // Surface level - // transfer updated prognostics from work arrays - mam_coupling::convert_work_arrays_to_mmr(vmr, vmrcw, q, qqcw); - mam_coupling::transfer_work_arrays_to_prognostics(q, qqcw, progs, k); - }); - }); + // loop over atmosphere columns and compute aerosol microphyscs + Kokkos::parallel_for( + "MAMMicrophysics::run_impl", policy, + KOKKOS_LAMBDA(const ThreadTeam &team) { + const int icol = team.league_rank(); // column index + const Real col_lat = col_latitudes(icol); // column latitude (degrees?) + + // convert column latitude to radians + const Real rlats = col_lat * M_PI / 180.0; + + // fetch column-specific atmosphere state data + const auto atm = mam_coupling::atmosphere_for_column(dry_atm, icol); + const auto wet_diameter_icol = + ekat::subview(wet_geometric_mean_diameter_i, icol); + const auto dry_diameter_icol = + ekat::subview(dry_geometric_mean_diameter_i, icol); + const auto wetdens_icol = ekat::subview(wetdens, icol); + + // fetch column-specific subviews into aerosol prognostics + mam4::Prognostics progs = + mam_coupling::aerosols_for_column(dry_aero, icol); + + const auto invariants_icol = ekat::subview(invariants, icol); + mam4::mo_setext::Forcing forcings_in[extcnt]; + + for(int i = 0; i < extcnt; ++i) { + const int nsectors = forcings[i].nsectors; + const int frc_ndx = forcings[i].frc_ndx; + const auto file_alt_data = forcings[i].file_alt_data; + + forcings_in[i].nsectors = nsectors; + forcings_in[i].frc_ndx = frc_ndx; + // We may need to move this line where we read files. + forcings_in[i].file_alt_data = file_alt_data; + for(int isec = 0; isec < forcings[i].nsectors; ++isec) { + const auto field = elevated_emis_output[isec + forcings[i].offset]; + forcings_in[i].fields_data[isec] = ekat::subview(field, icol); + } + } // extcnt for loop + + const auto extfrc_icol = ekat::subview(extfrc, icol); + + view_1d cnst_offline_icol[mam4::mo_setinv::num_tracer_cnst]; + for(int i = 0; i < mam4::mo_setinv::num_tracer_cnst; ++i) { + cnst_offline_icol[i] = ekat::subview(cnst_offline[i], icol); + } + + // calculate o3 column densities (first component of col_dens in Fortran + // code) + auto o3_col_dens_i = ekat::subview(o3_col_dens, icol); + const auto &work_photo_table_icol = + ekat::subview(work_photo_table, icol); + + const auto &photo_rates_icol = ekat::subview(photo_rates, icol); + + const auto linoz_o3_clim_icol = ekat::subview(linoz_o3_clim, icol); + const auto linoz_t_clim_icol = ekat::subview(linoz_t_clim, icol); + const auto linoz_o3col_clim_icol = + ekat::subview(linoz_o3col_clim, icol); + const auto linoz_PmL_clim_icol = ekat::subview(linoz_PmL_clim, icol); + const auto linoz_dPmL_dO3_icol = ekat::subview(linoz_dPmL_dO3, icol); + const auto linoz_dPmL_dT_icol = ekat::subview(linoz_dPmL_dT, icol); + const auto linoz_dPmL_dO3col_icol = + ekat::subview(linoz_dPmL_dO3col, icol); + const auto linoz_cariolle_pscs_icol = + ekat::subview(linoz_cariolle_pscs, icol); + const auto nevapr_icol = ekat::subview(nevapr, icol); + const auto prain_icol = ekat::subview(prain, icol); + const auto work_set_het_icol = ekat::subview(work_set_het, icol); + + // Surface temperature + const Real sfc_air_temp = atm.temperature(surface_lev); + + // Surface specific humidity + const Real sfc_spec_hum = atm.vapor_mixing_ratio(surface_lev); + + // Surface potential temperature + //(FIXME: We followed Fortran, compare it with MAM4xx's potential temp + // func) + const Real sfc_potential_temp = sfc_air_temp * (1.0 + sfc_spec_hum); + + // Surface pressure at 10m (Followed the fortran code) + const Real pressure_10m = dry_atm.p_mid(icol, surface_lev); + + // Wind speed at the surface + const Real wind_speed = + haero::sqrt(u_wind(icol, surface_lev) * u_wind(icol, surface_lev) + + v_wind(icol, surface_lev) * v_wind(icol, surface_lev)); + + // Total rain at the surface + const Real rain = precip_liq_surf_mass(icol, surface_lev) + + precip_ice_surf_mass(icol, surface_lev); + + // Snow depth on land [m] + const Real snow_height = snow_depth_land(icol); + + // Downwelling solar flux at the surface (value at interface) [w/m2] + const Real solar_flux = sw_flux_dn(icol, surface_lev + 1); + + Real fraction_landuse_icol[mam4::mo_drydep::n_land_type]; + for(int i = 0; i < mam4::mo_drydep::n_land_type; ++i) { + fraction_landuse_icol[i] = fraction_landuse(icol, i); + } + + // ????? FIXME: We should get its value after the rebase + const int col_index_season[mam4::mo_drydep::n_land_type] = { + 1, 2, 3, 4, 5, 6, 7, 8, 9}; + // These output values need to be put somewhere: + Real dvel[gas_pcnst] = {}; // deposition velocity [1/cm/s] + Real dflx[gas_pcnst] = {}; // deposition flux [1/cm^2/s] + + // Output: values are dvel, dvlx + // Input/Output: progs::stateq, progs::qqcw + mam4::microphysics::perform_atmospheric_chemistry_and_microphysics( + team, dt, rlats, month, sfc_temperature(icol), sfc_air_temp, + sfc_potential_temp, sfc_pressure(icol), pressure_10m, sfc_spec_hum, + wind_speed, rain, snow_height, solar_flux, cnst_offline_icol, + forcings_in, atm, photo_table, chlorine_loading, config.setsox, + config.amicphys, config.linoz.psc_T, zenith_angle(icol), + d_sfc_alb_dir_vis(icol), o3_col_dens_i, photo_rates_icol, + extfrc_icol, invariants_icol, work_photo_table_icol, + linoz_o3_clim_icol, linoz_t_clim_icol, linoz_o3col_clim_icol, + linoz_PmL_clim_icol, linoz_dPmL_dO3_icol, linoz_dPmL_dT_icol, + linoz_dPmL_dO3col_icol, linoz_cariolle_pscs_icol, eccf, + adv_mass_kg_per_moles, fraction_landuse_icol, + + col_index_season, // FIXME: Get it after Changes sync with E3SM + + clsmap_4, permute_4, offset_aerosol, config.linoz.o3_sfc, + config.linoz.o3_tau, config.linoz.o3_lbl, dry_diameter_icol, + wet_diameter_icol, wetdens_icol, dry_atm.phis(icol), cmfdqr, + prain_icol, nevapr_icol, work_set_het_icol, drydep_data, dvel, dflx, + progs); + + // Update constituent fluxes with gas drydep fluxes (dflx) + // FIXME: Possible units mismatch (dflx is in kg/cm2/s but + // constituent_fluxes is kg/m2/s) (Following mimics Fortran code + // behavior but we should look into it) + for(int ispc = offset_aerosol; ispc < mam4::pcnst; ++ispc) { + constituent_fluxes(icol, ispc) = dflx[ispc - offset_aerosol]; + } + }); // parallel_for for the column loop + Kokkos::fence(); // postprocess output Kokkos::parallel_for("postprocess", policy, postprocess_); Kokkos::fence(); -} -void MAMMicrophysics::finalize_impl() { -} +} // MAMMicrophysics::run_impl -} // namespace scream +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp index 5f5a44b846be..532e30423014 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp @@ -4,25 +4,12 @@ #include #include #include - -#include "impl/mam4_amicphys.cpp" // mam4xx top-level microphysics function(s) - -#include -#include +#include "readfiles/tracer_reader_utils.hpp" +// For calling MAM4 processes #include - #include -#ifndef KOKKOS_ENABLE_CUDA -#define protected_except_cuda public -#define private_except_cuda public -#else -#define protected_except_cuda protected -#define private_except_cuda private -#endif - -namespace scream -{ +namespace scream { // The process responsible for handling MAM4 aerosol microphysics. The AD // stores exactly ONE instance of this class in its list of subcomponents. @@ -31,28 +18,22 @@ class MAMMicrophysics final : public scream::AtmosphereProcess { using KT = ekat::KokkosTypes; // views for single- and multi-column data - using view_1d_int = typename KT::template view_1d; using view_1d = typename KT::template view_1d; using view_2d = typename KT::template view_2d; + using view_3d = typename KT::template view_3d; using const_view_1d = typename KT::template view_1d; using const_view_2d = typename KT::template view_2d; - // unmanaged views (for buffer and workspace manager) - using uview_1d = Unmanaged>; - using uview_2d = Unmanaged>; + using view_1d_host = typename KT::view_1d::HostMirror; - // a quantity stored in a single vertical column with a single index - using ColumnView = mam4::ColumnView; + using view_int_2d = typename KT::template view_2d; // a thread team dispatched to a single vertical column using ThreadTeam = mam4::ThreadTeam; -public: - + public: // Constructor - MAMMicrophysics(const ekat::Comm& comm, const ekat::ParameterList& params); - -protected_except_cuda: + MAMMicrophysics(const ekat::Comm &comm, const ekat::ParameterList ¶ms); // -------------------------------------------------------------------------- // AtmosphereProcess overrides (see share/atm_process/atmosphere_process.hpp) @@ -60,59 +41,59 @@ class MAMMicrophysics final : public scream::AtmosphereProcess { // process metadata AtmosphereProcessType type() const override; - std::string name() const override; - // set aerosol microphysics configuration parameters (called by constructor) - void configure(const ekat::ParameterList& params); + // The name of the subcomponent + std::string name() const { return "mam_aero_microphysics"; } // grid - void set_grids(const std::shared_ptr grids_manager) override; + void set_grids( + const std::shared_ptr grids_manager) override; // management of common atm process memory size_t requested_buffer_size_in_bytes() const override; void init_buffers(const ATMBufferManager &buffer_manager) override; - // process behavior + // Initialize variables void initialize_impl(const RunType run_type) override; - void run_impl(const double dt) override; - void finalize_impl() override; - // performs some checks on the tracers group - void set_computed_group_impl(const FieldGroup& group) override; + // Run the process by one time step + void run_impl(const double dt) override; -private_except_cuda: + // Finalize + void finalize_impl(){/*Do nothing*/}; + private: // number of horizontal columns and vertical levels int ncol_, nlev_; - // configuration data (for the moment, we plan to be able to move this to - // the device, so we can't use C++ strings) + // The orbital year, used for zenith angle calculations: + // If > 0, use constant orbital year for duration of simulation + // If < 0, use year from timestamp for orbital parameters + int m_orbital_year; + + // Orbital parameters, used for zenith angle calculations. + // If >= 0, bypass computation based on orbital year and use fixed parameters + // If < 0, compute based on orbital year, specified above + // These variables are required to be double. + double m_orbital_eccen; // Eccentricity + double m_orbital_obliq; // Obliquity + double m_orbital_mvelp; // Vernal Equinox Mean Longitude of Perihelion + struct Config { - // photolysis parameters - struct { - char rsf_file[MAX_FILENAME_LEN]; - char xs_long_file[MAX_FILENAME_LEN]; - } photolysis; // stratospheric chemistry parameters struct { - int o3_lbl; // number of layers with ozone decay from the surface - int o3_sfc; // set from namelist input linoz_sfc - int o3_tau; // set from namelist input linoz_tau - Real psc_T; // set from namelist input linoz_psc_T - char chlorine_loading_file[MAX_FILENAME_LEN]; + int o3_lbl; // number of layers with ozone decay from the surface + Real o3_sfc; // set from namelist input linoz_sfc + Real o3_tau; // set from namelist input linoz_tau + Real psc_T; // set from namelist input linoz_psc_T } linoz; // aqueous chemistry parameters mam4::mo_setsox::Config setsox; // aero microphysics configuration (see impl/mam4_amicphys.cpp) - impl::AmicPhysConfig amicphys; - - // dry deposition parameters - struct { - char srf_file[MAX_FILENAME_LEN]; - } drydep; + mam4::microphysics::AmicPhysConfig amicphys; }; Config config_; @@ -124,39 +105,41 @@ class MAMMicrophysics final : public scream::AtmosphereProcess { // on host: initializes preprocess functor with necessary state data void initialize(const int ncol, const int nlev, - const mam_coupling::WetAtmosphere& wet_atm, - const mam_coupling::AerosolState& wet_aero, - const mam_coupling::DryAtmosphere& dry_atm, - const mam_coupling::AerosolState& dry_aero) { - ncol_ = ncol; - nlev_ = nlev; - wet_atm_ = wet_atm; - wet_aero_ = wet_aero; - dry_atm_ = dry_atm; - dry_aero_ = dry_aero; + const mam_coupling::WetAtmosphere &wet_atm, + const mam_coupling::AerosolState &wet_aero, + const mam_coupling::DryAtmosphere &dry_atm, + const mam_coupling::AerosolState &dry_aero) { + ncol_pre_ = ncol; + nlev_pre_ = nlev; + wet_atm_pre_ = wet_atm; + wet_aero_pre_ = wet_aero; + dry_atm_pre_ = dry_atm; + dry_aero_pre_ = dry_aero; } KOKKOS_INLINE_FUNCTION - void operator()(const Kokkos::TeamPolicy::member_type& team) const { - const int i = team.league_rank(); // column index - - compute_vertical_layer_heights(team, dry_atm_, i); - team.team_barrier(); // allows kernels below to use layer heights - compute_updraft_velocities(team, wet_atm_, dry_atm_, i); - compute_dry_mixing_ratios(team, wet_atm_, dry_atm_, i); - compute_dry_mixing_ratios(team, wet_atm_, wet_aero_, dry_aero_, i); + void operator()( + const Kokkos::TeamPolicy::member_type &team) const { + const int i = team.league_rank(); // column index + + compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); + compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, + dry_aero_pre_, i); team.team_barrier(); - } // operator() + + compute_vertical_layer_heights(team, dry_atm_pre_, i); + compute_updraft_velocities(team, wet_atm_pre_, dry_atm_pre_, i); + } // operator() // number of horizontal columns and vertical levels - int ncol_, nlev_; + int ncol_pre_, nlev_pre_; // local atmospheric and aerosol state data - mam_coupling::WetAtmosphere wet_atm_; - mam_coupling::DryAtmosphere dry_atm_; - mam_coupling::AerosolState wet_aero_, dry_aero_; + mam_coupling::WetAtmosphere wet_atm_pre_; + mam_coupling::DryAtmosphere dry_atm_pre_; + mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; - }; // MAMMicrophysics::Preprocess + }; // MAMMicrophysics::Preprocess // Postprocessing functor struct Postprocess { @@ -164,33 +147,35 @@ class MAMMicrophysics final : public scream::AtmosphereProcess { // on host: initializes postprocess functor with necessary state data void initialize(const int ncol, const int nlev, - const mam_coupling::WetAtmosphere& wet_atm, - const mam_coupling::AerosolState& wet_aero, - const mam_coupling::DryAtmosphere& dry_atm, - const mam_coupling::AerosolState& dry_aero) { - ncol_ = ncol; - nlev_ = nlev; - wet_atm_ = wet_atm; - wet_aero_ = wet_aero; - dry_atm_ = dry_atm; - dry_aero_ = dry_aero; + const mam_coupling::WetAtmosphere &wet_atm, + const mam_coupling::AerosolState &wet_aero, + const mam_coupling::DryAtmosphere &dry_atm, + const mam_coupling::AerosolState &dry_aero) { + ncol_post_ = ncol; + nlev_post_ = nlev; + wet_atm_post_ = wet_atm; + wet_aero_post_ = wet_aero; + dry_atm_post_ = dry_atm; + dry_aero_post_ = dry_aero; } KOKKOS_INLINE_FUNCTION - void operator()(const Kokkos::TeamPolicy::member_type& team) const { - const int i = team.league_rank(); // column index - compute_wet_mixing_ratios(team, dry_atm_, dry_aero_, wet_aero_, i); + void operator()( + const Kokkos::TeamPolicy::member_type &team) const { + const int i = team.league_rank(); // column index + compute_wet_mixing_ratios(team, dry_atm_post_, dry_aero_post_, + wet_aero_post_, i); team.team_barrier(); - } // operator() + } // operator() // number of horizontal columns and vertical levels - int ncol_, nlev_; + int ncol_post_, nlev_post_; // local atmospheric and aerosol state data - mam_coupling::WetAtmosphere wet_atm_; - mam_coupling::DryAtmosphere dry_atm_; - mam_coupling::AerosolState wet_aero_, dry_aero_; - }; // MAMMicrophysics::Postprocess + mam_coupling::WetAtmosphere wet_atm_post_; + mam_coupling::DryAtmosphere dry_atm_post_; + mam_coupling::AerosolState wet_aero_post_, dry_aero_post_; + }; // MAMMicrophysics::Postprocess // MAM4 aerosol particle size description mam4::AeroConfig aero_config_; @@ -202,29 +187,68 @@ class MAMMicrophysics final : public scream::AtmosphereProcess { // atmospheric and aerosol state variables mam_coupling::WetAtmosphere wet_atm_; mam_coupling::DryAtmosphere dry_atm_; - mam_coupling::AerosolState wet_aero_, dry_aero_; + mam_coupling::AerosolState wet_aero_, dry_aero_; // photolysis rate table (column-independent) mam4::mo_photo::PhotoTableData photo_table_; // column areas, latitudes, longitudes - const_view_1d col_areas_, col_latitudes_, col_longitudes_; + const_view_1d col_latitudes_; - // time step number - int step_; + // surface albedo: shortwave, direct + const_view_1d d_sfc_alb_dir_vis_; // workspace manager for internal local variables - //ekat::WorkspaceManager workspace_mgr_; + // ekat::WorkspaceManager workspace_mgr_; mam_coupling::Buffer buffer_; // physics grid for column information std::shared_ptr grid_; - // sets defaults for "namelist parameters" - void set_defaults_(); - -}; // MAMMicrophysics - -} // namespace scream - -#endif // EAMXX_MAM_MICROPHYSICS_HPP + mam_coupling::TracerTimeState linoz_time_state_; + view_2d work_photo_table_; + std::vector chlorine_values_; + std::vector chlorine_time_secs_; + view_3d photo_rates_; + + // invariants members + mam_coupling::TracerTimeState trace_time_state_; + std::shared_ptr TracerDataReader_; + std::shared_ptr TracerHorizInterp_; + mam_coupling::TracerData tracer_data_; + view_3d invariants_; + std::string oxid_file_name_; + view_2d cnst_offline_[4]; + + // linoz reader + std::shared_ptr LinozDataReader_; + std::shared_ptr LinozHorizInterp_; + mam_coupling::TracerData linoz_data_; + std::string linoz_file_name_; + + // Vertical emission uses 9 files, here I am using std::vector to stote + // instance of each file. + mam_coupling::TracerTimeState elevated_emiss_time_state_; + std::vector> ElevatedEmissionsDataReader_; + std::vector> ElevatedEmissionsHorizInterp_; + std::vector extfrc_lst_; + std::vector elevated_emis_data_; + std::map elevated_emis_file_name_; + std::map> elevated_emis_var_names_; + view_2d elevated_emis_output_[mam_coupling::MAX_NUM_ELEVATED_EMISSIONS_FIELDS]; + view_3d extfrc_; + mam_coupling::ForcingHelper forcings_[mam4::gas_chemistry::extcnt]; + + view_1d_host acos_cosine_zenith_host_; + view_1d acos_cosine_zenith_; + + view_int_2d index_season_lai_; + // // dq/dt for convection [kg/kg/s] + view_1d cmfdqr_; + view_2d work_set_het_; + +}; // MAMMicrophysics + +} // namespace scream + +#endif // EAMXX_MAM_MICROPHYSICS_HPP diff --git a/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp index 442a6500f215..68153624b635 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp @@ -55,13 +55,13 @@ void MAMOptics::set_grids( add_field("p_int", scalar3d_int, Pa, grid_name); // total pressure add_field("pseudo_density", scalar3d_mid, Pa, grid_name); add_field("pseudo_density_dry", scalar3d_mid, Pa, grid_name); - add_field("qv", scalar3d_mid, kg/kg, grid_name,"tracers"); // specific humidity - add_field("qi", scalar3d_mid, kg/kg, grid_name,"tracers"); // ice wet mixing ratio - add_field("ni", scalar3d_mid, n_unit, grid_name,"tracers"); // ice number mixing ratio + add_tracer("qv", grid_, kg/kg); // specific humidity + add_tracer("qi", grid_, kg/kg); // ice wet mixing ratio + add_tracer("ni", grid_, n_unit); // ice number mixing ratio // droplet activation can alter cloud liquid and number mixing ratios - add_field("qc", scalar3d_mid, kg/kg, grid_name,"tracers"); // cloud liquid wet mixing ratio - add_field("nc", scalar3d_mid, n_unit, grid_name,"tracers"); // cloud liquid wet number mixing ratio + add_tracer("qc", grid_, kg/kg); // cloud liquid wet mixing ratio + add_tracer("nc", grid_, n_unit); // cloud liquid wet number mixing ratio add_field("phis", scalar2d, m2 / s2, grid_name); add_field("cldfrac_tot", scalar3d_mid, nondim,grid_name); // cloud fraction @@ -86,12 +86,12 @@ void MAMOptics::set_grids( for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); - add_field(int_nmr_field_name, scalar3d_mid, n_unit,grid_name, "tracers"); + add_tracer(int_nmr_field_name, grid_, n_unit); for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); if(strlen(int_mmr_field_name) > 0) { - add_field(int_mmr_field_name, scalar3d_mid, kg/kg,grid_name, "tracers"); + add_tracer(int_mmr_field_name, grid_, kg/kg); } } } @@ -113,7 +113,7 @@ void MAMOptics::set_grids( // aerosol-related gases: mass mixing ratios for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - add_field(gas_mmr_field_name, scalar3d_mid, kg/kg, grid_name, "tracers"); + add_tracer(gas_mmr_field_name, grid_, kg/kg); } } @@ -140,7 +140,7 @@ void MAMOptics::initialize_impl(const RunType run_type) { wet_atm_.nc = get_field_in("nc").get_view(); wet_atm_.qi = get_field_in("qi").get_view(); wet_atm_.ni = get_field_in("ni").get_view(); - + constexpr int ntot_amode = mam4::AeroConfig::num_modes(); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_functions.hpp new file mode 100644 index 000000000000..9c01daf8223c --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_functions.hpp @@ -0,0 +1,73 @@ +#ifndef EAMXX_MAM_SRF_AND_ONLINE_EMISSIONS_FUNCTIONS_HPP +#define EAMXX_MAM_SRF_AND_ONLINE_EMISSIONS_FUNCTIONS_HPP + +namespace scream { + +namespace { + +using KT = ekat::KokkosTypes; +using view_1d = typename KT::template view_1d; +using view_2d = typename KT::template view_2d; +using const_view_1d = typename KT::template view_1d; +using const_view_2d = typename KT::template view_2d; + +//-------- Inititlize gas and aerosol fluxes ------ +void init_fluxes(const int &ncol, + view_2d &constituent_fluxes) { // input-output + + constexpr int pcnst = mam4::aero_model::pcnst; + const int gas_start_ind = mam4::utils::gasses_start_ind(); + + const auto policy = + ekat::ExeSpaceUtils::get_default_team_policy( + ncol, pcnst - gas_start_ind); + + // Parallel loop over all the columns + Kokkos::parallel_for( + policy, KOKKOS_LAMBDA(const KT::MemberType &team) { + const int icol = team.league_rank(); + view_1d flux_col = ekat::subview(constituent_fluxes, icol); + + // Zero out constituent fluxes only for gasses and aerosols + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, gas_start_ind, pcnst), + [&](int icnst) { flux_col(icnst) = 0; }); + }); +} // init_fluxes ends + +//-------- compute online emissions for dust, sea salt and marine organics ----- +void compute_online_dust_nacl_emiss( + const int &ncol, const int &nlev, const const_view_1d &ocnfrac, + const const_view_1d &sst, const const_view_2d &u_wind, + const const_view_2d &v_wind, const const_view_2d &dstflx, + const const_view_1d &mpoly, const const_view_1d &mprot, + const const_view_1d &mlip, const const_view_1d &soil_erodibility, + const const_view_2d &z_mid, + // output + view_2d &constituent_fluxes) { + const int surf_lev = nlev - 1; // surface level + + Kokkos::parallel_for( + "online_emis_fluxes", ncol, KOKKOS_LAMBDA(int icol) { + // Input + const const_view_1d dstflx_icol = ekat::subview(dstflx, icol); + + // Output + view_1d fluxes_col = ekat::subview(constituent_fluxes, icol); + + // Compute online emissions + // NOTE: mam4::aero_model_emissions calculates mass and number emission + // fluxes in units of [kg/m2/s or #/m2/s] (MKS), so no need to convert + mam4::aero_model_emissions::aero_model_emissions( + sst(icol), ocnfrac(icol), u_wind(icol, surf_lev), + v_wind(icol, surf_lev), z_mid(icol, surf_lev), dstflx_icol, + soil_erodibility(icol), mpoly(icol), mprot(icol), mlip(icol), + // out + fluxes_col); + }); +} // compute_online_dust_nacl_emiss ends + +} // namespace +} // namespace scream + +#endif // EAMXX_MAM_SRF_AND_ONLINE_EMISSIONS_FUNCTIONS_HPP diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 850a82d0896d..fd3ebf79700f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -1,14 +1,24 @@ -//#include #include +// For surface and online emission functions +#include + +// For reading soil erodibility file +#include + namespace scream { +// For reading soil erodibility file +using soilErodibilityFunc = + soil_erodibility::soilErodibilityFunctions; + // ================================================================ // Constructor // ================================================================ MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms) : AtmosphereProcess(comm, params) { + // FIXME: Do we want to read dust emiss factor from the namelist?? /* Anything that can be initialized without grid information can be * initialized here. Like universal constants. */ @@ -26,18 +36,98 @@ void MAMSrfOnlineEmiss::set_grids( nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column using namespace ekat::units; + constexpr auto m2 = pow(m, 2); + constexpr auto s2 = pow(s, 2); + constexpr auto q_unit = kg / kg; // units of mass mixing ratios of tracers + constexpr auto n_unit = 1 / kg; // units of number mixing ratios of tracers + constexpr auto nondim = ekat::units::Units::nondimensional(); - static constexpr int pcnst = mam4::aero_model::pcnst; - const FieldLayout scalar2d_pcnct = - grid_->get_2d_vector_layout(pcnst, "num_phys_constituents"); + const FieldLayout scalar2d = grid_->get_2d_scalar_layout(); + const FieldLayout scalar3d_m = grid_->get_3d_scalar_layout(true); // mid + const FieldLayout scalar3d_i = grid_->get_3d_scalar_layout(false); // int + + // For U and V components of wind + const FieldLayout vector3d = grid_->get_3d_vector_layout(true, 2); + + // For components of dust flux + const FieldLayout vector4d = grid_->get_2d_vector_layout(4); + + // -------------------------------------------------------------------------- + // These variables are "Required" or pure inputs for the process + // -------------------------------------------------------------------------- + + // ----------- Atmospheric quantities ------------- + + // -- Variables required for building DS to compute z_mid -- + // Specific humidity [kg/kg] + // FIXME: Comply with add_tracer calls + add_field("qv", scalar3d_m, q_unit, grid_name, "tracers"); + + // Cloud liquid mass mixing ratio [kg/kg] + add_field("qc", scalar3d_m, q_unit, grid_name, "tracers"); + + // Cloud ice mass mixing ratio [kg/kg] + add_field("qi", scalar3d_m, q_unit, grid_name, "tracers"); + + // Cloud liquid number mixing ratio [1/kg] + add_field("nc", scalar3d_m, n_unit, grid_name, "tracers"); + + // Cloud ice number mixing ratio [1/kg] + add_field("ni", scalar3d_m, n_unit, grid_name, "tracers"); + + // Temperature[K] at midpoints + add_field("T_mid", scalar3d_m, K, grid_name); + + // Vertical pressure velocity [Pa/s] at midpoints + add_field("omega", scalar3d_m, Pa / s, grid_name); + + // Total pressure [Pa] at midpoints + add_field("p_mid", scalar3d_m, Pa, grid_name); + + // Total pressure [Pa] at interfaces + add_field("p_int", scalar3d_i, Pa, grid_name); + + // Layer thickness(pdel) [Pa] at midpoints + add_field("pseudo_density", scalar3d_m, Pa, grid_name); + + // Planetary boundary layer height [m] + add_field("pbl_height", scalar2d, m, grid_name); + + // Surface geopotential [m2/s2] + add_field("phis", scalar2d, m2 / s2, grid_name); + + //----------- Variables from microphysics scheme ------------- + + // Total cloud fraction [fraction] (Require only for building DS) + add_field("cldfrac_tot", scalar3d_m, nondim, grid_name); + + // -- Variables required for online dust and sea salt emissions -- + + // U and V components of the wind[m/s] + add_field("horiz_winds", vector3d, m / s, grid_name); + + //----------- Variables from coupler (ocean component)--------- + // Ocean fraction [unitless] + add_field("ocnfrac", scalar2d, nondim, grid_name); + + // Sea surface temperature [K] + add_field("sst", scalar2d, K, grid_name); + + // dust fluxes [kg/m^2/s]: Four flux values for each column + add_field("dstflx", vector4d, kg / m2 / s, grid_name); // ------------------------------------------------------------- - // These variables are "Computed" or outputs for the process + // These variables are "Updated" or input-outputs for the process // ------------------------------------------------------------- - static constexpr Units m2(m * m, "m2"); + + constexpr int pcnst = mam4::aero_model::pcnst; + const FieldLayout vector2d_pcnst = + grid_->get_2d_vector_layout(pcnst, "num_phys_constituents"); + // Constituent fluxes of species in [kg/m2/s] - add_field("constituent_fluxes", scalar2d_pcnct, kg / m2 / s, - grid_name); + // FIXME: confirm if it is Updated or Computed + add_field("constituent_fluxes", vector2d_pcnst, kg / m2 / s, + grid_name); // Surface emissions remapping file auto srf_map_file = m_params.get("srf_remap_file", ""); @@ -63,6 +153,7 @@ void MAMSrfOnlineEmiss::set_grids( so2.species_name = "so2"; so2.sectors = {"AGR", "RCO", "SHP", "SLV", "TRA", "WST"}; srf_emiss_species_.push_back(so2); // add to the vector + //-------------------------------------------------------------------- // Init bc_a4 srf emiss data structures //-------------------------------------------------------------------- @@ -147,7 +238,48 @@ void MAMSrfOnlineEmiss::set_grids( // output ispec_srf.horizInterp_, ispec_srf.data_start_, ispec_srf.data_end_, ispec_srf.data_out_, ispec_srf.dataReader_); - } + } // srf emissions file read init + + // ------------------------------------------------------------- + // Setup to enable reading soil erodibility file + // ------------------------------------------------------------- + + const std::string soil_erodibility_data_file = + m_params.get("soil_erodibility_file"); + + // Field to be read from file + const std::string soil_erod_fld_name = "mbl_bsn_fct_geo"; + + // Dimensions of the field + const std::string soil_erod_dname = "ncol"; + + // initialize the file read + soilErodibilityFunc::init_soil_erodibility_file_read( + ncol_, soil_erod_fld_name, soil_erod_dname, grid_, + soil_erodibility_data_file, srf_map_file, serod_horizInterp_, + serod_dataReader_); // output + + // ------------------------------------------------------------- + // Setup to enable reading marine organics file + // ------------------------------------------------------------- + const std::string marine_organics_data_file = + m_params.get("marine_organics_file"); + + // Fields to be read from file (order matters as they are read in the same + // order) + const std::vector marine_org_fld_name = { + "TRUEPOLYC", "TRUEPROTC", "TRUELIPC"}; + + // Dimensions of the field + const std::string marine_org_dname = "ncol"; + + // initialize the file read + marineOrganicsFunc::init_marine_organics_file_read( + ncol_, marine_org_fld_name, marine_org_dname, grid_, + marine_organics_data_file, srf_map_file, + // output + morg_horizInterp_, morg_data_start_, morg_data_end_, morg_data_out_, + morg_dataReader_); } // set_grid ends @@ -182,6 +314,42 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { // INITIALIZE_IMPL // ================================================================ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { + // --------------------------------------------------------------- + // Input fields read in from IC file, namelist or other processes + // --------------------------------------------------------------- + + // Populate the wet atmosphere state with views from fields + wet_atm_.qv = get_field_in("qv").get_view(); + + // Following wet_atm vars are required only for building DS + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + + // Populate the dry atmosphere state with views from fields + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.p_int = get_field_in("p_int").get_view(); + + // Following dry_atm vars are required only for building DS + dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); + dry_atm_.pblh = get_field_in("pbl_height").get_view(); + dry_atm_.omega = get_field_in("omega").get_view(); + + // store fields converted to dry mmr from wet mmr in dry_atm_ + dry_atm_.z_mid = buffer_.z_mid; + dry_atm_.z_iface = buffer_.z_iface; + dry_atm_.dz = buffer_.dz; + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; + dry_atm_.w_updraft = buffer_.w_updraft; + dry_atm_.z_surf = 0.0; // FIXME: for now + // --------------------------------------------------------------- // Output fields // --------------------------------------------------------------- @@ -212,10 +380,27 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { ispec_srf.data_end_); // output } + //----------------------------------------------------------------- + // Read Soil erodibility data + //----------------------------------------------------------------- + // This data is time-independent, we read all data here for the + // entire simulation + soilErodibilityFunc::update_soil_erodibility_data_from_file( + serod_dataReader_, *serod_horizInterp_, + soil_erodibility_); // output + + //-------------------------------------------------------------------- + // Update marine orgaincs from file + //-------------------------------------------------------------------- + // Time dependent data + marineOrganicsFunc::update_marine_organics_data_from_file( + morg_dataReader_, timestamp(), curr_month, *morg_horizInterp_, + morg_data_end_); // output + //----------------------------------------------------------------- // Setup preprocessing and post processing //----------------------------------------------------------------- - preprocess_.initialize(constituent_fluxes_); + preprocess_.initialize(ncol_, nlev_, wet_atm_, dry_atm_); } // end initialize_impl() @@ -223,14 +408,79 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMSrfOnlineEmiss::run_impl(const double dt) { - // Zero-out output - Kokkos::deep_copy(preprocess_.constituent_fluxes_pre_, 0); + const auto scan_policy = ekat::ExeSpaceUtils< + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + + // preprocess input -- needs a scan for the calculation of atm height + Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + Kokkos::fence(); + + // Constituent fluxes [kg/m^2/s] + auto constituent_fluxes = this->constituent_fluxes_; + // Zero out constituent fluxes only for gasses and aerosols + init_fluxes(ncol_, // in + constituent_fluxes); // in-out + Kokkos::fence(); // Gather time and state information for interpolation - auto ts = timestamp() + dt; + const auto ts = timestamp() + dt; //-------------------------------------------------------------------- - // Interpolate srf emiss data + // Online emissions from dust and sea salt + //-------------------------------------------------------------------- + + // --- Interpolate marine organics data -- + + // Update TimeState, note the addition of dt + morg_timeState_.t_now = ts.frac_of_year_in_days(); + + // Update time state and if the month has changed, update the data. + marineOrganicsFunc::update_marine_organics_timestate( + morg_dataReader_, ts, *morg_horizInterp_, + // output + morg_timeState_, morg_data_start_, morg_data_end_); + + // Call the main marine organics routine to get interpolated forcings. + marineOrganicsFunc::marineOrganics_main(morg_timeState_, morg_data_start_, + morg_data_end_, morg_data_out_); + + // Marine organics emission data read from the file (order is important here) + const const_view_1d mpoly = ekat::subview(morg_data_out_.emiss_sectors, 0); + const const_view_1d mprot = ekat::subview(morg_data_out_.emiss_sectors, 1); + const const_view_1d mlip = ekat::subview(morg_data_out_.emiss_sectors, 2); + + // Ocean fraction [unitless] + const const_view_1d ocnfrac = + get_field_in("ocnfrac").get_view(); + + // Sea surface temperature [K] + const const_view_1d sst = get_field_in("sst").get_view(); + + // U wind component [m/s] + const const_view_2d u_wind = + get_field_in("horiz_winds").get_component(0).get_view(); + + // V wind component [m/s] + const const_view_2d v_wind = + get_field_in("horiz_winds").get_component(1).get_view(); + + // Dust fluxes [kg/m^2/s]: Four flux values for each column + const const_view_2d dstflx = get_field_in("dstflx").get_view(); + + // Soil edodibility [fraction] + const const_view_1d soil_erodibility = this->soil_erodibility_; + + // Vertical layer height at midpoints + const const_view_2d z_mid = dry_atm_.z_mid; + + compute_online_dust_nacl_emiss(ncol_, nlev_, ocnfrac, sst, u_wind, v_wind, + dstflx, mpoly, mprot, mlip, soil_erodibility, + z_mid, + // output + constituent_fluxes); + Kokkos::fence(); + //-------------------------------------------------------------------- + // Interpolate srf emiss data read in from emissions files //-------------------------------------------------------------------- for(srf_emiss_ &ispec_srf : srf_emiss_species_) { @@ -256,20 +506,18 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // modify units from molecules/cm2/s to kg/m2/s auto fluxes_in_mks_units = this->fluxes_in_mks_units_; - auto constituent_fluxes = this->constituent_fluxes_; const Real mfactor = amufac * mam4::gas_chemistry::adv_mass[species_index - offset_]; + const view_1d ispec_outdata0 = + ekat::subview(ispec_srf.data_out_.emiss_sectors, 0); // Parallel loop over all the columns to update units Kokkos::parallel_for( - "fluxes", ncol_, KOKKOS_LAMBDA(int icol) { - fluxes_in_mks_units(icol) = - ispec_srf.data_out_.emiss_sectors(0, icol) * mfactor; + "srf_emis_fluxes", ncol_, KOKKOS_LAMBDA(int icol) { + fluxes_in_mks_units(icol) = ispec_outdata0(icol) * mfactor; constituent_fluxes(icol, species_index) = fluxes_in_mks_units(icol); }); - } // for loop for species Kokkos::fence(); -} // run_imple ends - +} // run_impl ends // ============================================================================= } // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 031fb62d8b75..1a3bb4f36e3f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -8,11 +8,12 @@ #include #include +// For reading marine organics file +#include + // For declaring surface and online emission class derived from atm process // class #include - -// #include #include namespace scream { @@ -20,19 +21,31 @@ namespace scream { // The process responsible for handling MAM4 surface and online emissions. The // AD stores exactly ONE instance of this class in its list of subcomponents. class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { - using KT = ekat::KokkosTypes; - using view_1d = typename KT::template view_1d; - using view_2d = typename KT::template view_2d; + using KT = ekat::KokkosTypes; + using view_1d = typename KT::template view_1d; + using view_2d = typename KT::template view_2d; + using const_view_1d = typename KT::template view_1d; + using const_view_2d = typename KT::template view_2d; // number of horizontal columns and vertical levels int ncol_, nlev_; + // Wet and dry states of atmosphere + mam_coupling::WetAtmosphere wet_atm_; + mam_coupling::DryAtmosphere dry_atm_; + // buffer for sotring temporary variables mam_coupling::Buffer buffer_; // physics grid for column information std::shared_ptr grid_; + // Sea surface temoerature [K] + const_view_1d sst_; + + // Dust fluxes (four values for each col) [kg/m2/s] + const_view_2d dust_fluxes_; + // Constituent fluxes of species in [kg/m2/s] view_2d constituent_fluxes_; @@ -42,8 +55,16 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // Unified atomic mass unit used for unit conversion (BAD constant) static constexpr Real amufac = 1.65979e-23; // 1.e4* kg / amu + // For reading soil erodibility file + std::shared_ptr serod_horizInterp_; + std::shared_ptr serod_dataReader_; + const_view_1d soil_erodibility_; + public: + // For reading surface emissions and marine organics file using srfEmissFunc = mam_coupling::srfEmissFunctions; + using marineOrganicsFunc = + marine_organics::marineOrganicsFunctions; // Constructor MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms); @@ -80,13 +101,35 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { struct Preprocess { Preprocess() = default; // on host: initializes preprocess functor with necessary state data - void initialize(const view_2d &constituent_fluxes) { - constituent_fluxes_pre_ = constituent_fluxes; + void initialize(const int &ncol, const int &nlev, + const mam_coupling::WetAtmosphere &wet_atm, + const mam_coupling::DryAtmosphere &dry_atm) { + ncol_pre_ = ncol; + nlev_pre_ = nlev; + wet_atm_pre_ = wet_atm; + dry_atm_pre_ = dry_atm; } + KOKKOS_INLINE_FUNCTION + void operator()( + const Kokkos::TeamPolicy::member_type &team) const { + const int icol = team.league_rank(); // column index + + compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, icol); + team.team_barrier(); + // vertical heights has to be computed after computing dry mixing ratios + // for atmosphere + compute_vertical_layer_heights(team, dry_atm_pre_, icol); + compute_updraft_velocities(team, wet_atm_pre_, dry_atm_pre_, icol); + } // Preprocess operator() + // local variables for preprocess struct - view_2d constituent_fluxes_pre_; - }; // MAMSrfOnlineEmiss::Preprocess + // number of horizontal columns and vertical levels + int ncol_pre_, nlev_pre_; + // local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_pre_; + mam_coupling::DryAtmosphere dry_atm_pre_; + }; // MAMSrfOnlineEmiss::Preprocess private: // preprocessing scratch pad Preprocess preprocess_; @@ -95,9 +138,11 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // FIXME: Remove the hardwired indices and use a function // to find them from an array. const std::map spcIndex_in_pcnst_ = { - {"so2", 12}, {"dms", 13}, {"so4_a1", 15}, - {"num_a1", 22}, {"so4_a2", 23}, {"num_a2", 27}, - {"pom_a4", 36}, {"bc_a4", 37}, {"num_a4", 39}}; + {"so2", 12}, {"dms", 13}, {"so4_a1", 15}, {"dst_a1", 19}, + {"ncl_a1", 20}, {"mom_a1", 21}, {"num_a1", 22}, {"so4_a2", 23}, + {"ncl_a2", 25}, {"mom_a2", 26}, {"num_a2", 27}, {"dst_a3", 28}, + {"ncl_a3", 29}, {"num_a3", 35}, {"pom_a4", 36}, {"bc_a4", 37}, + {"mom_a4", 38}, {"num_a4", 39}}; // A struct carrying all the fields needed to read // surface emissions of a species @@ -122,6 +167,13 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // A vector for carrying emissions for all the species std::vector srf_emiss_species_; + // For reading marine organics file + std::shared_ptr morg_horizInterp_; + std::shared_ptr morg_dataReader_; + marineOrganicsFunc::marineOrganicsTimeState morg_timeState_; + marineOrganicsFunc::marineOrganicsInput morg_data_start_, morg_data_end_; + marineOrganicsFunc::marineOrganicsOutput morg_data_out_; + // offset for converting pcnst index to gas_pcnst index static constexpr int offset_ = mam4::aero_model::pcnst - mam4::gas_chemistry::gas_pcnst; diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 2a81c1a9c427..0295a23cb4d1 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -28,8 +28,8 @@ void MAMWetscav::set_grids( // The units of mixing ratio Q are technically non-dimensional. // Nevertheless, for output reasons, we like to see 'kg/kg'. - auto q_unit = kg / kg; - auto n_unit = 1 / kg; // units of number mixing ratios of tracers + auto q_unit = kg / kg; + auto n_unit = 1 / kg; // units of number mixing ratios of tracers m_grid = grids_manager->get_grid("Physics"); const auto &grid_name = m_grid->name(); @@ -61,19 +61,19 @@ void MAMWetscav::set_grids( // ----------- Atmospheric quantities ------------- // Specific humidity [kg/kg] - add_field("qv", scalar3d_mid, q_unit, grid_name, "tracers"); + add_tracer("qv", m_grid, q_unit); // cloud liquid mass mixing ratio [kg/kg] - add_field("qc", scalar3d_mid, q_unit, grid_name, "tracers"); + add_tracer("qc", m_grid, q_unit); // cloud ice mass mixing ratio [kg/kg] - add_field("qi", scalar3d_mid, q_unit, grid_name, "tracers"); + add_tracer("qi", m_grid, q_unit); // cloud liquid number mixing ratio [1/kg] - add_field("nc", scalar3d_mid, n_unit, grid_name, "tracers"); + add_tracer("nc", m_grid, n_unit); // cloud ice number mixing ratio [1/kg] - add_field("ni", scalar3d_mid, n_unit, grid_name, "tracers"); + add_tracer("ni", m_grid, n_unit); // Temperature[K] at midpoints add_field("T_mid", scalar3d_mid, K, grid_name); @@ -161,8 +161,7 @@ void MAMWetscav::set_grids( // interstitial aerosol tracers of interest: number (n) mixing ratios const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(imode); - add_field(int_nmr_field_name, scalar3d_mid, n_unit, grid_name, - "tracers"); + add_tracer(int_nmr_field_name, m_grid, n_unit); // cloudborne aerosol tracers of interest: number (n) mixing ratios // Note: Do *not* add cld borne aerosols to the "tracer" group as these are @@ -177,8 +176,7 @@ void MAMWetscav::set_grids( const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(imode, ispec); if(strlen(int_mmr_field_name) > 0) { - add_field(int_mmr_field_name, scalar3d_mid, q_unit, grid_name, - "tracers"); + add_tracer(int_mmr_field_name, m_grid, q_unit); } // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios @@ -198,8 +196,7 @@ void MAMWetscav::set_grids( // aerosol-related gases: mass mixing ratios for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - add_field(gas_mmr_field_name, scalar3d_mid, q_unit, grid_name, - "tracers"); + add_tracer(gas_mmr_field_name, m_grid, q_unit); } // ------------------------------------------------------------- @@ -208,7 +205,7 @@ void MAMWetscav::set_grids( static constexpr auto m3 = m * m * m; // Aerosol dry particle diameter [m] - add_field("dgncur_a", scalar3d_mid_nmodes, m, grid_name); + add_field("dgnum", scalar3d_mid_nmodes, m, grid_name); // Wet aerosol density [kg/m3] add_field("wetdens", scalar3d_mid_nmodes, kg / m3, grid_name); @@ -478,7 +475,7 @@ void MAMWetscav::run_impl(const double dt) { const auto wet_geometric_mean_diameter_i = get_field_out("dgnumwet").get_view(); const auto dry_geometric_mean_diameter_i = - get_field_out("dgncur_a").get_view(); + get_field_out("dgnum").get_view(); const auto qaerwat = get_field_out("qaerwat").get_view(); const auto wetdens = get_field_out("wetdens").get_view(); @@ -542,12 +539,19 @@ void MAMWetscav::run_impl(const double dt) { auto wetdens_icol = ekat::subview(wetdens, icol); const auto prain_icol = ekat::subview(prain, icol); + Real scavimptblnum[mam4::aero_model::nimptblgrow_total] + [mam4::AeroConfig::num_modes()]; + Real scavimptblvol[mam4::aero_model::nimptblgrow_total] + [mam4::AeroConfig::num_modes()]; + + mam4::wetdep::init_scavimptbl(scavimptblvol, scavimptblnum); + mam4::wetdep::aero_model_wetdep( team, atm, progs, tends, dt, // inputs cldt_icol, rprdsh_icol, rprddp_icol, evapcdp_icol, evapcsh_icol, dp_frac_icol, sh_frac_icol, icwmrdp_col, icwmrsh_icol, nevapr_icol, - dlf_icol, prain_icol, + dlf_icol, prain_icol, scavimptblnum, scavimptblvol, // outputs wet_diameter_icol, dry_diameter_icol, qaerwat_icol, wetdens_icol, aerdepwetis_icol, aerdepwetcw_icol, work_icol); diff --git a/components/eamxx/src/physics/mam/impl/README.md b/components/eamxx/src/physics/mam/impl/README.md deleted file mode 100644 index f05a484a82aa..000000000000 --- a/components/eamxx/src/physics/mam/impl/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# MAM4 Integration Code - -This folder contains C++ implementations of the higher-level MAM4 interface -routines for aerosol microphysics, cloud-aerosol interactions, and optical -properties. We've retained the overall structure of the original Fortran code -to make it easier to understand for folks who are more familiar with the -original implementation of MAM4. - -## Contents - -* `mam4_amicphys.cpp` - high-level MAM4 microphysics interface code diff --git a/components/eamxx/src/physics/mam/impl/compute_o3_column_density.cpp b/components/eamxx/src/physics/mam/impl/compute_o3_column_density.cpp deleted file mode 100644 index ceb992bc8863..000000000000 --- a/components/eamxx/src/physics/mam/impl/compute_o3_column_density.cpp +++ /dev/null @@ -1,43 +0,0 @@ -namespace scream::impl { - -KOKKOS_INLINE_FUNCTION -void compute_o3_column_density(const ThreadTeam& team, const haero::Atmosphere& atm, - const mam4::Prognostics &progs, ColumnView o3_col_dens) { - constexpr int gas_pcnst = mam4::gas_chemistry::gas_pcnst; // number of gas phase species - constexpr int nfs = mam4::gas_chemistry::nfs; // number of "fixed species" - // constexpr Real mwdry = 1.0/haero::Constants::molec_weight_dry_air; - - Real o3_col_deltas[mam4::nlev+1] = {}; // o3 column density above model [1/cm^2] - // NOTE: if we need o2 column densities, set_ub_col and setcol must be changed - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, atm.num_levels()), [&](const int k) { - - // Real temp = atm.temperature(k); - // Real pmid = atm.pressure(k); - Real pdel = atm.hydrostatic_dp(k); - // Real qv = atm.vapor_mixing_ratio(k); - - // ... map incoming mass mixing ratios to working array - Real q[gas_pcnst], qqcw[gas_pcnst]; - mam_coupling::transfer_prognostics_to_work_arrays(progs, k, q, qqcw); - - // ... set atmosphere mean mass to the molecular weight of dry air - // and compute water vapor vmr - // Real mbar = mwdry; - // Real h2ovmr = mam4::conversions::vmr_from_mmr(qv, mbar); - - // ... Xform from mmr to vmr - Real vmr[gas_pcnst], vmrcw[gas_pcnst]; - mam_coupling::convert_work_arrays_to_vmr(q, qqcw, vmr, vmrcw); - - // ... compute invariants for this level - Real invariants[nfs]; - // setinv(invariants, temp, h2ovmr, vmr, pmid); FIXME: not yet ported - - // compute the change in o3 density for this column above its neighbor - mam4::mo_photo::set_ub_col(o3_col_deltas[k+1], vmr, invariants, pdel); - }); - // sum the o3 column deltas to densities - mam4::mo_photo::setcol(o3_col_deltas, o3_col_dens); -} - -} // namespace scream::impl diff --git a/components/eamxx/src/physics/mam/impl/compute_water_content.cpp b/components/eamxx/src/physics/mam/impl/compute_water_content.cpp deleted file mode 100644 index ea7190afff91..000000000000 --- a/components/eamxx/src/physics/mam/impl/compute_water_content.cpp +++ /dev/null @@ -1,99 +0,0 @@ -#include - -namespace scream::impl { - -KOKKOS_INLINE_FUNCTION -void compute_water_content(const mam4::Prognostics &progs, int k, - Real qv, Real temp, Real pmid, - Real dgncur_a[mam4::AeroConfig::num_modes()], - Real dgncur_awet[mam4::AeroConfig::num_modes()], - Real wetdens[mam4::AeroConfig::num_modes()], - Real qaerwat[mam4::AeroConfig::num_modes()]) { - constexpr int num_modes = mam4::AeroConfig::num_modes(); - constexpr int num_aero_ids = mam4::AeroConfig::num_aerosol_ids(); - - // get some information about aerosol species - // FIXME: this isn't great! - constexpr int maxd_aspectype = mam4::water_uptake::maxd_aspectype; - int nspec_amode[num_modes], lspectype_amode[maxd_aspectype][num_modes]; - Real specdens_amode[maxd_aspectype], spechygro[maxd_aspectype]; - mam4::water_uptake::get_e3sm_parameters(nspec_amode, lspectype_amode, - specdens_amode, spechygro); - - // extract aerosol tracers for this level into state_q, which is needed - // for computing dry aerosol properties below - // FIXME: we should eliminate this index translation stuff - constexpr int nvars = aero_model::pcnst; - Real state_q[nvars]; // aerosol tracers for level k - for (int imode = 0; imode < num_modes; ++imode) { - int la, lc; // interstitial and cloudborne indices within state_q - - // number mixing ratios - mam4::convproc::assign_la_lc(imode, -1, la, lc); - state_q[la] = progs.n_mode_i[imode](k); - state_q[lc] = progs.n_mode_c[imode](k); - // aerosol mass mixing ratios - for (int iaero = 0; iaero < num_aero_ids; ++iaero) { - mam4::convproc::assign_la_lc(imode, iaero, la, lc); - auto mode = static_cast(imode); - auto aero = static_cast(iaero); - int ispec = mam4::aerosol_index_for_mode(mode, aero); - if (ispec != -1) { - state_q[la] = progs.q_aero_i[imode][ispec](k); - state_q[lc] = progs.q_aero_c[imode][ispec](k); - } - } - } - - // compute the dry volume for each mode, and from it the current dry - // geometric nominal particle diameter. - // FIXME: We have to do some gymnastics here to set up the calls to - // FIXME: calcsize. This could be improved. - Real inv_densities[num_modes][num_aero_ids] = {}; - for (int imode = 0; imode < num_modes; ++imode) { - const int n_spec = mam4::num_species_mode(imode); - for (int ispec = 0; ispec < n_spec; ++ispec) { - const int iaer = static_cast(mam4::mode_aero_species(imode, ispec)); - const Real density = mam4::aero_species(iaer).density; - inv_densities[imode][ispec] = 1.0 / density; - } - } - for (int imode = 0; imode < num_modes; ++imode) { - Real dryvol_i, dryvol_c; // interstitial and cloudborne dry volumes - mam4::calcsize::compute_dry_volume_k(k, imode, inv_densities, progs, - dryvol_i, dryvol_c); - - // NOTE: there's some disagreement over whether vol2num should be called - // NOTE: num2vol here, so I'm just adopting the nomenclature used by - // NOTE: the following call to calcsize) - const mam4::Mode& mode = mam4::modes(imode); - Real vol2num_min = 1.0/mam4::conversions::mean_particle_volume_from_diameter( - mode.max_diameter, mode.mean_std_dev); - Real vol2num_max = 1.0/mam4::conversions::mean_particle_volume_from_diameter( - mode.min_diameter, mode.mean_std_dev); - Real vol2num; - mam4::calcsize::update_diameter_and_vol2num(dryvol_i, - progs.n_mode_i[imode](k), vol2num_min, vol2num_max, - mode.min_diameter, mode.max_diameter, mode.mean_std_dev, - dgncur_a[imode], vol2num); - } - - // calculate dry aerosol properties - Real hygro[num_modes], naer[num_modes], dryrad[num_modes], - dryvol[num_modes], drymass[num_modes], - rhcrystal[num_modes], rhdeliques[num_modes], specdens_1[num_modes]; - mam4::water_uptake::modal_aero_water_uptake_dryaer(nspec_amode, specdens_amode, - spechygro, lspectype_amode, state_q, dgncur_a, hygro, - naer, dryrad, dryvol, drymass, rhcrystal, rhdeliques, specdens_1); - - // calculate wet aerosol properties - Real rh = mam4::conversions::relative_humidity_from_vapor_mixing_ratio(qv, temp, pmid); - Real wetrad[num_modes], wetvol[num_modes], wtrvol[num_modes]; - mam4::water_uptake::modal_aero_water_uptake_wetaer(rhcrystal, rhdeliques, dgncur_a, - dryrad, hygro, rh, naer, dryvol, wetrad, wetvol, wtrvol, dgncur_awet, - qaerwat); - mam4::water_uptake::modal_aero_water_uptake_wetdens(wetvol, wtrvol, - drymass, specdens_1, wetdens); -} - -} // namespace scream::impl diff --git a/components/eamxx/src/physics/mam/impl/gas_phase_chemistry.cpp b/components/eamxx/src/physics/mam/impl/gas_phase_chemistry.cpp deleted file mode 100644 index 5401fa5e3da2..000000000000 --- a/components/eamxx/src/physics/mam/impl/gas_phase_chemistry.cpp +++ /dev/null @@ -1,381 +0,0 @@ -#include - -namespace scream::impl { - -using mam4::utils::min_max_bound; - -using HostView1D = haero::DeviceType::view_1d::HostMirror; -using HostViewInt1D = haero::DeviceType::view_1d::HostMirror; - -//------------------------------------------------------------------------- -// Reading the photolysis table -//------------------------------------------------------------------------- -// This logic is currently implemented using serial NetCDF calls for -// clarity of purpose. We should probably read the data for the photolysis -// table using SCREAM's SCORPIO interface instead, but I wanted to make -// clear what we're trying to do in terms of "elementary" operations first. - -// ON HOST (MPI root rank only), reads the dimension of a NetCDF variable from -// the file with the given ID -int nc_dimension(const char *file, int nc_id, const char *dim_name) { - int dim_id; - int result = nc_inq_dimid(nc_id, dim_name, &dim_id); - EKAT_REQUIRE_MSG(result == 0, "Error! Couldn't fetch " << dim_name << - " dimension ID from NetCDF file '" << file << "'\n"); - size_t dim; - result = nc_inq_dimlen(nc_id, dim_id, &dim); - EKAT_REQUIRE_MSG(result == 0, "Error! Couldn't fetch " << dim_name << - " dimension from NetCDF file '" << file << "'\n"); - return static_cast(dim); -} - -// ON HOST (MPI root rank only), reads data from the given NetCDF variable from -// the file with the given ID into the given Kokkos host View -template -void read_nc_var(const char *file, int nc_id, const char *var_name, V host_view) { - int var_id; - int result = nc_inq_varid(nc_id, var_name, &var_id); - EKAT_REQUIRE_MSG(result == 0, "Error! Couldn't fetch ID for variable '" << var_name << - "' from NetCDF file '" << file << "'\n"); - result = nc_get_var(nc_id, var_id, host_view.data()); - EKAT_REQUIRE_MSG(result == 0, "Error! Couldn't read data for variable '" << var_name << - "' from NetCDF file '" << file << "'\n"); -} - -// ON HOST (MPI root rank only), reads data from the NetCDF variable with the -// given ID, from the file with the given ID, into the given Kokkos host View -template -void read_nc_var(const char *file, int nc_id, int var_id, V host_view) { - int result = nc_get_var(nc_id, var_id, host_view.data()); - EKAT_REQUIRE_MSG(result == 0, "Error! Couldn't read data for variable with ID " << - var_id << " from NetCDF file '" << file << "'\n"); -} - -// ON HOST (MPI root only), sets the lng_indexer and pht_alias_mult_1 host views -// according to parameters in our (hardwired) chemical mechanism -void set_lng_indexer_and_pht_alias_mult_1(const char *file, int nc_id, - HostViewInt1D lng_indexer, - HostView1D pht_alias_mult_1) { - // NOTE: it seems that the chemical mechanism we're using - // NOTE: 1. sets pht_alias_lst to a blank string [1] - // NOTE: 2. sets pht_alias_mult_1 to 1.0 [1] - // NOTE: 3. sets rxt_tag_lst to ['jh2o2', 'usr_HO2_HO2', 'usr_SO2_OH', 'usr_DMS_OH'] [2] - // NOTE: References: - // NOTE: [1] (https://github.com/eagles-project/e3sm_mam4_refactor/blob/refactor-maint-2.0/components/eam/src/chemistry/pp_linoz_mam4_resus_mom_soag/mo_sim_dat.F90#L117) - // NOTE: [2] (https://github.com/eagles-project/e3sm_mam4_refactor/blob/refactor-maint-2.0/components/eam/src/chemistry/pp_linoz_mam4_resus_mom_soag/mo_sim_dat.F90#L99) - - // populate lng_indexer (see https://github.com/eagles-project/e3sm_mam4_refactor/blob/refactor-maint-2.0/components/eam/src/chemistry/mozart/mo_jlong.F90#L180) - static const char *var_names[4] = {"jh2o2", "usr_HO2_HO2", "usr_SO2_OH", "usr_DMS_OH"}; - for (int m = 0; m < mam4::mo_photo::phtcnt; ++m) { - int var_id; - int result = nc_inq_varid(nc_id, var_names[m], &var_id); - EKAT_REQUIRE_MSG(result == 0, "Error! Couldn't fetch ID for variable '" - << var_names[m] << "' from NetCDF file '" << file << "'\n"); - lng_indexer(m) = var_id; - } - - // set pht_alias_mult_1 to 1 - Kokkos::deep_copy(pht_alias_mult_1, 1.0); -} - -// ON HOST (MPI root only), populates the etfphot view using rebinned -// solar data from our solar_data_file -void populate_etfphot(HostView1D we, HostView1D etfphot) { - // FIXME: It looks like EAM is relying on a piece of infrastructure that - // FIXME: we just don't have in EAMxx (eam/src/chemistry/utils/solar_data.F90). - // FIXME: I have no idea whether EAMxx has a plan for supporting this - // FIXME: solar irradiance / photon flux data, and I'm not going to recreate - // FIXME: that capability here. So this is an unplugged hole. - // FIXME: - // FIXME: If we are going to do this the way EAM does it, the relevant logic - // FIXME: is the call to rebin() in eam/src/chemistry/mozart/mo_jlong.F90, - // FIXME: around line 104. - - // FIXME: zero the photon flux for now - Kokkos::deep_copy(etfphot, 0); -} - -// ON HOST, reads the photolysis table (used for gas phase chemistry) from the -// files with the given names -mam4::mo_photo::PhotoTableData read_photo_table(const ekat::Comm& comm, - const char *rsf_file, - const char* xs_long_file) { - // NOTE: at the time of development, SCREAM's SCORPIO interface seems intended - // NOTE: for domain-decomposed grid data. The files we're reading here are not - // NOTE: spatial data, and should be the same everywhere, so we read them - // NOTE: using serial NetCDF calls on MPI rank 0 and broadcast to other ranks. - const int mpi_root = 0; - int rsf_id, xs_long_id; // NetCDF file IDs (used only on MPI root) - int nw, nump, numsza, numcolo3, numalb, nt, np_xs; // table dimensions - if (comm.rank() == mpi_root) { // read dimension data from files and broadcast - // open files - int result = nc_open(rsf_file, NC_NOWRITE, &rsf_id); - EKAT_REQUIRE_MSG(result == 0, "Error! Couldn't open rsf_file '" << rsf_file << "'\n"); - result = nc_open(xs_long_file, NC_NOWRITE, &xs_long_id); - EKAT_REQUIRE_MSG(result == 0, "Error! Couldn't open xs_long_file '" << xs_long_file << "'\n"); - - // read and broadcast dimension data - nump = nc_dimension(rsf_file, rsf_id, "numz"); - numsza = nc_dimension(rsf_file, rsf_id, "numsza"); - numalb = nc_dimension(rsf_file, rsf_id, "numalb"); - numcolo3 = nc_dimension(rsf_file, rsf_id, "numcolo3fact"); - nt = nc_dimension(xs_long_file, xs_long_id, "numtemp"); - nw = nc_dimension(xs_long_file, xs_long_id, "numwl"); - np_xs = nc_dimension(xs_long_file, xs_long_id, "numprs"); - - int dim_data[7] = {nump, numsza, numcolo3, numalb, nt, nw, np_xs}; - comm.broadcast(dim_data, 7, mpi_root); - } else { // receive broadcasted dimension data from root rank - int dim_data[7]; - comm.broadcast(dim_data, 7, mpi_root); - nump = dim_data[0]; - numsza = dim_data[1]; - numcolo3 = dim_data[2]; - numalb = dim_data[3]; - nt = dim_data[4]; - nw = dim_data[5]; - np_xs = dim_data[6]; - } - - // set up the lng_indexer and pht_alias_mult_1 views based on our - // (hardwired) chemical mechanism - HostViewInt1D lng_indexer_h("lng_indexer(host)", mam4::mo_photo::phtcnt); - HostView1D pht_alias_mult_1_h("pht_alias_mult_1(host)", 2); - if (comm.rank() == mpi_root) { - set_lng_indexer_and_pht_alias_mult_1(xs_long_file, xs_long_id, - lng_indexer_h, pht_alias_mult_1_h); - } - comm.broadcast(lng_indexer_h.data(), mam4::mo_photo::phtcnt, mpi_root); - comm.broadcast(pht_alias_mult_1_h.data(), 2, mpi_root); - - // compute the size of the foremost dimension of xsqy using lng_indexer - int numj = 0; - for (int m = 0; m < mam4::mo_photo::phtcnt; ++m) { - if (lng_indexer_h(m) > 0) { - for (int mm = 0; mm < m; ++mm) { - if (lng_indexer_h(mm) == lng_indexer_h(m)) { - break; - } - ++numj; - } - } - } - - // allocate the photolysis table - auto table = mam4::mo_photo::create_photo_table_data(nw, nt, np_xs, numj, - nump, numsza, numcolo3, - numalb); - - // allocate host views for table data - auto rsf_tab_h = Kokkos::create_mirror_view(table.rsf_tab); - auto xsqy_h = Kokkos::create_mirror_view(table.xsqy); - auto sza_h = Kokkos::create_mirror_view(table.sza); - auto alb_h = Kokkos::create_mirror_view(table.alb); - auto press_h = Kokkos::create_mirror_view(table.press); - auto colo3_h = Kokkos::create_mirror_view(table.colo3); - auto o3rat_h = Kokkos::create_mirror_view(table.o3rat); - auto etfphot_h = Kokkos::create_mirror_view(table.etfphot); - auto prs_h = Kokkos::create_mirror_view(table.prs); - - if (comm.rank() == mpi_root) { // read data from files and broadcast - // read file data into our host views - read_nc_var(rsf_file, rsf_id, "pm", press_h); - read_nc_var(rsf_file, rsf_id, "sza", sza_h); - read_nc_var(rsf_file, rsf_id, "alb", alb_h); - read_nc_var(rsf_file, rsf_id, "colo3fact", o3rat_h); - read_nc_var(rsf_file, rsf_id, "colo3", colo3_h); - read_nc_var(rsf_file, rsf_id, "RSF", rsf_tab_h); - - read_nc_var(xs_long_file, xs_long_id, "pressure", prs_h); - - // read xsqy data (using lng_indexer_h for the first index) - int ndx = 0; - for (int m = 0; m < mam4::mo_photo::phtcnt; ++m) { - if (lng_indexer_h(m) > 0) { - auto xsqy_ndx_h = ekat::subview(xsqy_h, ndx); - read_nc_var(xs_long_file, xs_long_id, lng_indexer_h(m), xsqy_ndx_h); - ++ndx; - } - } - - // populate etfphot by rebinning solar data - HostView1D wc_h("wc", nw), wlintv_h("wlintv", nw), we_h("we", nw+1); - read_nc_var(rsf_file, rsf_id, "wc", wc_h); - read_nc_var(rsf_file, rsf_id, "wlintv", wlintv_h); - for (int i = 0; i < nw; ++i) { - we_h(i) = wc_h(i) - 0.5 * wlintv_h(i); - } - we_h(nw) = wc_h(nw-1) - 0.5 * wlintv_h(nw-1); - populate_etfphot(we_h, etfphot_h); - - // close the files - nc_close(rsf_id); - nc_close(xs_long_id); - } - - // broadcast host views from MPI root to others - comm.broadcast(rsf_tab_h.data(), nw*numalb*numcolo3*numsza*nump, mpi_root); - comm.broadcast(xsqy_h.data(), numj*nw*nt*np_xs, mpi_root); - comm.broadcast(sza_h.data(), numsza, mpi_root); - comm.broadcast(alb_h.data(), numalb, mpi_root); - comm.broadcast(press_h.data(), nump, mpi_root); - comm.broadcast(o3rat_h.data(), numcolo3, mpi_root); - comm.broadcast(colo3_h.data(), nump, mpi_root); - comm.broadcast(etfphot_h.data(), nw, mpi_root); - comm.broadcast(prs_h.data(), np_xs, mpi_root); - - // copy host photolysis table into place on device - Kokkos::deep_copy(table.rsf_tab, rsf_tab_h); - Kokkos::deep_copy(table.xsqy, xsqy_h); - Kokkos::deep_copy(table.sza, sza_h); - Kokkos::deep_copy(table.alb, alb_h); - Kokkos::deep_copy(table.press, press_h); - Kokkos::deep_copy(table.colo3, colo3_h); - Kokkos::deep_copy(table.o3rat, o3rat_h); - Kokkos::deep_copy(table.etfphot, etfphot_h); - Kokkos::deep_copy(table.prs, prs_h); - Kokkos::deep_copy(table.pht_alias_mult_1, pht_alias_mult_1_h); - Kokkos::deep_copy(table.lng_indexer, lng_indexer_h); - - // compute gradients (on device) - Kokkos::parallel_for("del_p", nump-1, KOKKOS_LAMBDA(int i) { - table.del_p(i) = 1.0/::abs(table.press(i)- table.press(i+1)); - }); - Kokkos::parallel_for("del_sza", numsza-1, KOKKOS_LAMBDA(int i) { - table.del_sza(i) = 1.0/(table.sza(i+1) - table.sza(i)); - }); - Kokkos::parallel_for("del_alb", numalb-1, KOKKOS_LAMBDA(int i) { - table.del_alb(i) = 1.0/(table.alb(i+1) - table.alb(i)); - }); - Kokkos::parallel_for("del_o3rat", numcolo3-1, KOKKOS_LAMBDA(int i) { - table.del_o3rat(i) = 1.0/(table.o3rat(i+1) - table.o3rat(i)); - }); - Kokkos::parallel_for("dprs", np_xs-1, KOKKOS_LAMBDA(int i) { - table.dprs(i) = 1.0/(table.prs(i) - table.prs(i+1)); - }); - - return table; -} - -// performs gas phase chemistry calculations on a single level of a single -// atmospheric column -KOKKOS_INLINE_FUNCTION -void gas_phase_chemistry(Real zm, Real zi, Real phis, Real temp, Real pmid, Real pdel, Real dt, - const Real photo_rates[mam4::mo_photo::phtcnt], // in - const Real extfrc[mam4::gas_chemistry::extcnt], // in - Real q[mam4::gas_chemistry::gas_pcnst], // VMRs, inout - Real invariants[mam4::gas_chemistry::nfs]) { // out - // constexpr Real rga = 1.0/haero::Constants::gravity; - // constexpr Real m2km = 0.01; // converts m -> km - - // The following things are chemical mechanism dependent! See mam4xx/src/mam4xx/gas_chem_mechanism.hpp) - constexpr int gas_pcnst = mam4::gas_chemistry::gas_pcnst; // number of gas phase species - constexpr int rxntot = mam4::gas_chemistry::rxntot; // number of chemical reactions - constexpr int extcnt = mam4::gas_chemistry::extcnt; // number of species with external forcing - constexpr int indexm = 0; // FIXME: index of total atm density in invariants array - - constexpr int phtcnt = mam4::mo_photo::phtcnt; // number of photolysis reactions - - constexpr int itermax = mam4::gas_chemistry::itermax; - constexpr int clscnt4 = mam4::gas_chemistry::clscnt4; - - // NOTE: vvv these arrays were copied from mam4xx/gas_chem_mechanism.hpp vvv - constexpr int permute_4[gas_pcnst] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, - 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, - 20, 21, 22, 23, 24, 25, 26, 27, 28, 29}; - constexpr int clsmap_4[gas_pcnst] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, - 21, 22, 23, 24, 25, 26, 27, 28, 29, 30}; - - // These indices for species are fixed by the chemical mechanism - // std::string solsym[] = {"O3", "H2O2", "H2SO4", "SO2", "DMS", "SOAG", - // "so4_a1", "pom_a1", "soa_a1", "bc_a1", "dst_a1", - // "ncl_a1", "mom_a1", "num_a1", "so4_a2", "soa_a2", - // "ncl_a2", "mom_a2", "num_a2", "dst_a3", "ncl_a3", - // "so4_a3", "bc_a3", "pom_a3", "soa_a3", "mom_a3", - // "num_a3", "pom_a4", "bc_a4", "mom_a4", "num_a4"}; - constexpr int ndx_h2so4 = 2; - // std::string extfrc_list[] = {"SO2", "so4_a1", "so4_a2", "pom_a4", "bc_a4", - // "num_a1", "num_a2", "num_a3", "num_a4", "SOAG"}; - constexpr int synoz_ndx = -1; - - // fetch the zenith angle (not its cosine!) in degrees for this column. - // FIXME: For now, we fix the zenith angle. At length, we need to compute it - // FIXME: from EAMxx's current set of orbital parameters, which requires some - // FIXME: conversation with the EAMxx team. - - // xform geopotential height from m to km and pressure from Pa to mb - // Real zsurf = rga * phis; - // Real zmid = m2km * (zm + zsurf); - - // ... compute the column's invariants - // Real h2ovmr = q[0]; - // setinv(invariants, temp, h2ovmr, q, pmid); FIXME: not ported yet - - // ... set rates for "tabular" and user specified reactions - Real reaction_rates[rxntot]; - mam4::gas_chemistry::setrxt(reaction_rates, temp); - - // set reaction rates based on chemical invariants - // (indices (ndxes?) are taken from mam4 validation data and translated from - // 1-based indices to 0-based indices) - int usr_HO2_HO2_ndx = 1, usr_DMS_OH_ndx = 5, - usr_SO2_OH_ndx = 3, inv_h2o_ndx = 3; - mam4::gas_chemistry::usrrxt(reaction_rates, temp, invariants, invariants[indexm], - usr_HO2_HO2_ndx, usr_DMS_OH_ndx, - usr_SO2_OH_ndx, inv_h2o_ndx); - mam4::gas_chemistry::adjrxt(reaction_rates, invariants, invariants[indexm]); - - //=================================== - // Photolysis rates at time = t(n+1) - //=================================== - - // compute the rate of change from forcing - Real extfrc_rates[extcnt]; // [1/cm^3/s] - for (int mm = 0; mm < extcnt; ++mm) { - if (mm != synoz_ndx) { - extfrc_rates[mm] = extfrc[mm] / invariants[indexm]; - } - } - - // ... Form the washout rates - Real het_rates[gas_pcnst]; - // FIXME: not ported yet - //sethet(het_rates, pmid, zmid, phis, temp, cmfdqr, prain, nevapr, delt, - // invariants[indexm], q); - - - // save h2so4 before gas phase chem (for later new particle nucleation) - Real del_h2so4_gasprod = q[ndx_h2so4]; - - //=========================== - // Class solution algorithms - //=========================== - - // copy photolysis rates into reaction_rates (assumes photolysis rates come first) - for (int i = 0; i < phtcnt; ++i) { - reaction_rates[i] = photo_rates[i]; - } - - // ... solve for "Implicit" species - bool factor[itermax]; - for (int i = 0; i < itermax; ++i) { - factor[i] = true; - } - - // initialize error tolerances - Real epsilon[clscnt4]; - mam4::gas_chemistry::imp_slv_inti(epsilon); - - // solve chemical system implicitly - Real prod_out[clscnt4], loss_out[clscnt4]; - mam4::gas_chemistry::imp_sol(q, reaction_rates, het_rates, extfrc_rates, dt, - permute_4, clsmap_4, factor, epsilon, prod_out, loss_out); - - // save h2so4 change by gas phase chem (for later new particle nucleation) - if (ndx_h2so4 > 0) { - del_h2so4_gasprod = q[ndx_h2so4] - del_h2so4_gasprod; - } -} - -} // namespace scream::impl diff --git a/components/eamxx/src/physics/mam/impl/mam4_amicphys.cpp b/components/eamxx/src/physics/mam/impl/mam4_amicphys.cpp deleted file mode 100644 index 71cee1eeb407..000000000000 --- a/components/eamxx/src/physics/mam/impl/mam4_amicphys.cpp +++ /dev/null @@ -1,1769 +0,0 @@ -#include -#include -#include -#include -#include -#include - -namespace scream::impl { - -#define MAX_FILENAME_LEN 256 - -using namespace mam4; - -// number of constituents in gas chemistry "work arrays" -KOKKOS_INLINE_FUNCTION -constexpr int gas_pcnst() { - constexpr int gas_pcnst_ = mam4::gas_chemistry::gas_pcnst; - return gas_pcnst_; -} - -// number of aerosol/gas species tendencies -KOKKOS_INLINE_FUNCTION -constexpr int nqtendbb() { return 4; } - -// MAM4 aerosol microphysics configuration data -struct AmicPhysConfig { - // these switches activate various aerosol microphysics processes - bool do_cond; // condensation (a.k.a gas-aerosol exchange) - bool do_rename; // mode "renaming" - bool do_newnuc; // gas -> aerosol nucleation - bool do_coag; // aerosol coagulation - - // configurations for specific aerosol microphysics - mam4::GasAerExchProcess::ProcessConfig condensation; - mam4::NucleationProcess::ProcessConfig nucleation; - - // controls treatment of h2so4 condensation in mam_gasaerexch_1subarea - // 1 = sequential calc. of gas-chem prod then condensation loss - // 2 = simultaneous calc. of gas-chem prod and condensation loss - int gaexch_h2so4_uptake_optaa; - - // controls how nucleation interprets h2so4 concentrations - int newnuc_h2so4_conc_optaa; -}; - -namespace { - -KOKKOS_INLINE_FUNCTION constexpr int nqtendaa() { return 5; } -KOKKOS_INLINE_FUNCTION constexpr int nqqcwtendaa() { return 1; } -KOKKOS_INLINE_FUNCTION constexpr int nqqcwtendbb() { return 1; } -KOKKOS_INLINE_FUNCTION constexpr int iqtend_cond() { return 0; } -KOKKOS_INLINE_FUNCTION constexpr int iqtend_rnam() { return 1; } -KOKKOS_INLINE_FUNCTION constexpr int iqtend_nnuc() { return 2; } -KOKKOS_INLINE_FUNCTION constexpr int iqtend_coag() { return 3; } -KOKKOS_INLINE_FUNCTION constexpr int iqtend_cond_only() { return 4; } -KOKKOS_INLINE_FUNCTION constexpr int iqqcwtend_rnam() { return 0; } -KOKKOS_INLINE_FUNCTION constexpr int maxsubarea() { return 2; } - -// conversion factors -KOKKOS_INLINE_FUNCTION Real fcvt_gas(int gas_id) { - static const Real fcvt_gas_[AeroConfig::num_gas_ids()] = {1, 1, 1}; - return fcvt_gas_[gas_id]; -} -KOKKOS_INLINE_FUNCTION Real fcvt_aer(int aero_id) { - static const Real fcvt_aer_[AeroConfig::num_aerosol_ids()] = {1, 1, 1, 1, 1, 1, 1}; - return fcvt_aer_[aero_id]; -} - -// leave number mix-ratios unchanged (#/kmol-air) -KOKKOS_INLINE_FUNCTION Real fcvt_num() { return 1.0; } -// factor for converting aerosol water mix-ratios from (kg/kg) to (mol/mol) -KOKKOS_INLINE_FUNCTION Real fcvt_wtr() { return 1.0; } - -KOKKOS_INLINE_FUNCTION constexpr int lmapcc_val_nul() { return 0; } -KOKKOS_INLINE_FUNCTION constexpr int lmapcc_val_gas() { return 1; } -KOKKOS_INLINE_FUNCTION constexpr int lmapcc_val_aer() { return 2; } -KOKKOS_INLINE_FUNCTION constexpr int lmapcc_val_num() { return 3; } -KOKKOS_INLINE_FUNCTION int lmapcc_all(int index) { - static const int lmapcc_all_[gas_pcnst()] = { - lmapcc_val_nul(), lmapcc_val_gas(), lmapcc_val_nul(), lmapcc_val_nul(), - lmapcc_val_gas(), lmapcc_val_aer(), lmapcc_val_aer(), lmapcc_val_aer(), - lmapcc_val_aer(), lmapcc_val_aer(), lmapcc_val_aer(), lmapcc_val_aer(), - lmapcc_val_num(), lmapcc_val_aer(), lmapcc_val_aer(), lmapcc_val_aer(), - lmapcc_val_aer(), lmapcc_val_num(), lmapcc_val_aer(), lmapcc_val_aer(), - lmapcc_val_aer(), lmapcc_val_aer(), lmapcc_val_aer(), lmapcc_val_aer(), - lmapcc_val_aer(), lmapcc_val_num(), lmapcc_val_aer(), lmapcc_val_aer(), - lmapcc_val_aer(), lmapcc_val_num()}; - return lmapcc_all_[index]; -} - -// Where lmapcc_val_num are defined in lmapcc_all -KOKKOS_INLINE_FUNCTION int numptr_amode(int mode) { - static const int numptr_amode_[AeroConfig::num_modes()] = {12, 17, 25, 29}; - return numptr_amode_[mode]; -} - -// Where lmapcc_val_gas are defined in lmapcc_all -KOKKOS_INLINE_FUNCTION int lmap_gas(int mode) { - static const int lmap_gas_[AeroConfig::num_modes()] = {4, 1}; - return lmap_gas_[mode]; -} -// Where lmapcc_val_aer are defined in lmapcc_all -KOKKOS_INLINE_FUNCTION int lmassptr_amode(int aero_id, int mode) { - static const int lmassptr_amode_[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()] = { - {5, 13, 18, 26}, {6, 14, 19, 27}, {7, 15, 20, 28}, {8, 16, 21, -6}, - {9, -6, 22, -6}, {10, -6, 23, -6}, {11, -6, 24, -6}}; - return lmassptr_amode_[aero_id][mode]; -} - -KOKKOS_INLINE_FUNCTION -void subarea_partition_factors( - const Real - q_int_cell_avg, // in grid cell mean interstitial aerosol mixing ratio - const Real - q_cbn_cell_avg, // in grid cell mean cloud-borne aerosol mixing ratio - const Real fcldy, // in cloudy fraction of the grid cell - const Real fclea, // in clear fraction of the grid cell - Real &part_fac_q_int_clea, // out - Real &part_fac_q_int_cldy) // out -{ - // Calculate mixing ratios of each subarea - - // cloud-borne, cloudy subarea - const Real tmp_q_cbn_cldy = q_cbn_cell_avg / fcldy; - // interstitial, cloudy subarea - const Real tmp_q_int_cldy = - haero::max(0.0, ((q_int_cell_avg + q_cbn_cell_avg) - tmp_q_cbn_cldy)); - // interstitial, clear subarea - const Real tmp_q_int_clea = (q_int_cell_avg - fcldy * tmp_q_int_cldy) / fclea; - - // Calculate the corresponding paritioning factors for interstitial aerosols - // using the above-derived subarea mixing ratios plus the constraint that - // the cloud fraction weighted average of subarea mean need to match grid box - // mean. - - // *** question *** - // use same part_fac_q_int_clea/cldy for everything ? - // use one for number and one for all masses (based on total mass) ? - // use separate ones for everything ? - // maybe one for number and one for all masses is best, - // because number and mass have different activation fractions - // *** question *** - - Real tmp_aa = haero::max(1.e-35, tmp_q_int_clea * fclea) / - haero::max(1.e-35, q_int_cell_avg); - tmp_aa = haero::max(0.0, haero::min(1.0, tmp_aa)); - - part_fac_q_int_clea = tmp_aa / fclea; - part_fac_q_int_cldy = (1.0 - tmp_aa) / fcldy; -} - -KOKKOS_INLINE_FUNCTION -void construct_subareas_1gridcell( - const Real cld, // in - const Real relhumgcm, // in - const Real q_pregaschem[gas_pcnst()], // in q TMRs before - // gas-phase chemistry - const Real q_precldchem[gas_pcnst()], // in q TMRs before - // cloud chemistry - const Real qqcw_precldchem[gas_pcnst()], // in qqcw TMRs before - // cloud chemistry - const Real q[gas_pcnst()], // in current tracer mixing ratios (TMRs) - // *** MUST BE #/kmol-air for number - // *** MUST BE mol/mol-air for mass - const Real qqcw[gas_pcnst()], // in like q but for - // cloud-borner tracers - int &nsubarea, // out - int &ncldy_subarea, // out - int &jclea, // out - int &jcldy, // out - bool iscldy_subarea[maxsubarea()], // out - Real afracsub[maxsubarea()], // out - Real relhumsub[maxsubarea()], // out - Real qsub1[gas_pcnst()][maxsubarea()], // out interstitial - Real qsub2[gas_pcnst()][maxsubarea()], // out interstitial - Real qsub3[gas_pcnst()][maxsubarea()], // out interstitial - Real qqcwsub1[gas_pcnst()][maxsubarea()], // out cloud-borne - Real qqcwsub2[gas_pcnst()][maxsubarea()], // out cloud-borne - Real qqcwsub3[gas_pcnst()][maxsubarea()], // outcloud-borne - Real qaerwatsub3[AeroConfig::num_modes()] - [maxsubarea()], // out aerosol water mixing ratios (mol/mol) - Real qaerwat[AeroConfig::num_modes()] // in aerosol water mixing ratio - // (kg/kg, NOT mol/mol) -) { - static constexpr int num_modes = AeroConfig::num_modes(); - // cloud chemistry is only on when cld(i,k) >= 1.0e-5_wp - // it may be that the macrophysics has a higher threshold that this - const Real fcld_locutoff = 1.0e-5; - const Real fcld_hicutoff = 0.999; - - // qgcmN and qqcwgcmN (N=1:4) are grid-cell mean tracer mixing ratios (TMRs, - // mol/mol or #/kmol) - // N=1 - before gas-phase chemistry - // N=2 - before cloud chemistry - // N=3 - incoming values (before gas-aerosol exchange, newnuc, coag) - // qgcm1, qgcm2, qgcm3 - // qqcwgcm2, qqcwgcm3 - // qaerwatgcm3 ! aerosol water mixing ratios (mol/mol) - - // -------------------------------------------------------------------------------------- - // Determine the number of sub-areas, their fractional areas, and relative - // humidities - // -------------------------------------------------------------------------------------- - // if cloud fraction ~= 0, the grid-cell has a single clear sub-area - // (nsubarea = 1) if cloud fraction ~= 1, the grid-cell has a single cloudy - // sub-area (nsubarea = 1) otherwise, the grid-cell has a - // clear and a cloudy sub-area (nsubarea = 2) - - Real zfcldy = 0; - nsubarea = 0; - ncldy_subarea = 0; - jclea = 0; - jcldy = 0; - - if (cld < fcld_locutoff) { - nsubarea = 1; - jclea = 1; - } else if (cld > fcld_hicutoff) { - zfcldy = 1.0; - nsubarea = 1; - ncldy_subarea = 1; - jcldy = 1; - } else { - zfcldy = cld; - nsubarea = 2; - ncldy_subarea = 1; - jclea = 1; - jcldy = 2; - } - - const Real zfclea = 1.0 - zfcldy; - for (int i = 0; i < maxsubarea(); ++i) - iscldy_subarea[i] = false; - if (jcldy > 0) - iscldy_subarea[jcldy - 1] = true; - for (int i = 0; i < maxsubarea(); ++i) - afracsub[i] = 0.0; - if (jclea > 0) - afracsub[jclea - 1] = zfclea; - if (jcldy > 0) - afracsub[jcldy - 1] = zfcldy; - - // cldy_rh_sameas_clear is just to match mam_refactor. Compiler should - // optimize away. - const int cldy_rh_sameas_clear = 0; - if (ncldy_subarea <= 0) { - for (int i = 0; i < maxsubarea(); ++i) - relhumsub[i] = relhumgcm; - } else if (cldy_rh_sameas_clear > 0) { - for (int i = 0; i < maxsubarea(); ++i) - relhumsub[i] = relhumgcm; - } else { - if (jcldy > 0) { - relhumsub[jcldy - 1] = 1.0; - if (jclea > 0) { - const Real tmpa = - (relhumgcm - afracsub[jcldy - 1]) / afracsub[jclea - 1]; - relhumsub[jclea - 1] = haero::max(0.0, haero::min(1.0, tmpa)); - } - } - } - - // ---------------------------------------------------------------------------- - // Copy grid cell mean mixing ratios. - // These values, together with cloud fraction and a few assumptions, are used - // in the remainder of the subroutine to calculate the sub-area mean mixing - // ratios. - // ---------------------------------------------------------------------------- - // Interstitial aerosols - Real qgcm1[gas_pcnst()], qgcm2[gas_pcnst()], qgcm3[gas_pcnst()]; - for (int i = 0; i < gas_pcnst(); ++i) { - qgcm1[i] = haero::max(0.0, q_pregaschem[i]); - qgcm2[i] = haero::max(0.0, q_precldchem[i]); - qgcm3[i] = haero::max(0.0, q[i]); - } - - // Cloud-borne aerosols - Real qqcwgcm2[gas_pcnst()], qqcwgcm3[gas_pcnst()]; - for (int i = 0; i < gas_pcnst(); ++i) { - qqcwgcm2[i] = haero::max(0.0, qqcw_precldchem[i]); - qqcwgcm3[i] = haero::max(0.0, qqcw[i]); - } - - // aerosol water - Real qaerwatgcm3[num_modes] = {}; - for (int i = 0; i < num_modes; ++i) { - qaerwatgcm3[i] = haero::max(0.0, qaerwat[i]); - } - - // ---------------------------------------------------------------------------- - // Initialize the subarea mean mixing ratios - // ---------------------------------------------------------------------------- - { - const int n = haero::min(maxsubarea(), nsubarea + 1); - for (int i = 0; i < n; ++i) { - for (int j = 0; j < gas_pcnst(); ++j) { - qsub1[j][i] = 0.0; - qsub2[j][i] = 0.0; - qsub3[j][i] = 0.0; - qqcwsub1[j][i] = 0.0; - qqcwsub2[j][i] = 0.0; - qqcwsub3[j][i] = 0.0; - } - for (int j = 0; j < num_modes; ++j) { - qaerwatsub3[j][i] = 0.0; - } - } - } - - // ************************************************************************************************* - // Calculate initial (i.e., before cond/rnam/nnuc/coag) tracer mixing - // ratios within the sub-areas - // - for all-clear or all-cloudy cases, the sub-area TMRs are equal to the - // grid-cell means - // - for partly cloudy case, they are different. This is primarily - // because the - // interstitial aerosol mixing ratios are assumed lower in the cloudy - // sub-area than in the clear sub-area, because much of the aerosol is - // activated in the cloudy sub-area. - // ************************************************************************************************* - // Category I: partly cloudy case - // ************************************************************************************************* - if ((jclea > 0) && (jcldy > 0) && (jclea + jcldy == 3) && (nsubarea == 2)) { - - // --------------------------------------------------------------------- - // Set GAS mixing ratios in sub-areas (for the condensing gases only!!) - // --------------------------------------------------------------------- - for (int lmz = 0; lmz < gas_pcnst(); ++lmz) { - if (lmapcc_all(lmz) == lmapcc_val_gas()) { - - // assume gas in both sub-areas before gas-chem and cloud-chem equal - // grid-cell mean - for (int i = 0; i < nsubarea; ++i) { - qsub1[lmz][i] = qgcm1[lmz]; - qsub2[lmz][i] = qgcm2[lmz]; - } - // assume gas in clear sub-area after cloud-chem equals before - // cloud-chem value - qsub3[lmz][jclea - 1] = qsub2[lmz][jclea - 1]; - // gas in cloud sub-area then determined by grid-cell mean and clear - // values - qsub3[lmz][jcldy - 1] = - (qgcm3[lmz] - zfclea * qsub3[lmz][jclea - 1]) / zfcldy; - - // check that this does not produce a negative value - if (qsub3[lmz][jcldy - 1] < 0.0) { - qsub3[lmz][jcldy - 1] = 0.0; - qsub3[lmz][jclea - 1] = qgcm3[lmz] / zfclea; - } - } - } - // --------------------------------------------------------------------- - // Set CLOUD-BORNE AEROSOL mixing ratios in sub-areas. - // This is straightforward, as the same partitioning factors (0 or 1/f) - // are applied to all mass and number mixing ratios in all modes. - // --------------------------------------------------------------------- - // loop thru log-normal modes - for (int n = 0; n < num_modes; ++n) { - // number - then mass of individual species - of a mode - for (int l2 = -1; l2 < num_species_mode(n); ++l2) { - int lc; - if (l2 == -1) - lc = numptr_amode(n); - else - lc = lmassptr_amode(l2, n); - qqcwsub2[lc][jclea - 1] = 0.0; - qqcwsub2[lc][jcldy - 1] = qqcwgcm2[lc] / zfcldy; - qqcwsub3[lc][jclea - 1] = 0.0; - qqcwsub3[lc][jcldy - 1] = qqcwgcm3[lc] / zfcldy; - } - } - - // --------------------------------------------------------------------- - // Set INTERSTITIAL AEROSOL mixing ratios in sub-areas. - // --------------------------------------------------------------------- - for (int n = 0; n < num_modes; ++n) { - // ------------------------------------- - // Aerosol number - // ------------------------------------- - // grid cell mean, interstitial - Real tmp_q_cellavg_int = qgcm2[numptr_amode(n)]; - // grid cell mean, cloud-borne - Real tmp_q_cellavg_cbn = qqcwgcm2[numptr_amode(n)]; - - Real nmbr_part_fac_clea = 0; - Real nmbr_part_fac_cldy = 0; - subarea_partition_factors(tmp_q_cellavg_int, tmp_q_cellavg_cbn, zfcldy, - zfclea, nmbr_part_fac_clea, nmbr_part_fac_cldy); - - // Apply the partitioning factors to calculate sub-area mean number - // mixing ratios - - const int la = numptr_amode(n); - - qsub2[la][jclea - 1] = qgcm2[la] * nmbr_part_fac_clea; - qsub2[la][jcldy - 1] = qgcm2[la] * nmbr_part_fac_cldy; - qsub3[la][jclea - 1] = qgcm3[la] * nmbr_part_fac_clea; - qsub3[la][jcldy - 1] = qgcm3[la] * nmbr_part_fac_cldy; - - //------------------------------------- - // Aerosol mass - //------------------------------------- - // For aerosol mass, we use the total grid cell mean - // interstitial/cloud-borne mass mixing ratios to come up with the same - // partitioning for all species in the mode. - - // Compute the total mixing ratios by summing up the individual species - - tmp_q_cellavg_int = 0.0; // grid cell mean, interstitial - tmp_q_cellavg_cbn = 0.0; // grid cell mean, cloud-borne - - for (int l2 = 0; l2 < num_species_mode(n); ++l2) { - tmp_q_cellavg_int += qgcm2[lmassptr_amode(l2, n)]; - tmp_q_cellavg_cbn += qqcwgcm2[lmassptr_amode(l2, n)]; - } - Real mass_part_fac_clea = 0; - Real mass_part_fac_cldy = 0; - // Calculate the partitioning factors - subarea_partition_factors(tmp_q_cellavg_int, tmp_q_cellavg_cbn, zfcldy, - zfclea, mass_part_fac_clea, mass_part_fac_cldy); - - // Apply the partitioning factors to calculate sub-area mean mass mixing - // ratios - - for (int l2 = 0; l2 < num_species_mode(n); ++l2) { - const int la = lmassptr_amode(l2, n); - - qsub2[la][jclea - 1] = qgcm2[la] * mass_part_fac_clea; - qsub2[la][jcldy - 1] = qgcm2[la] * mass_part_fac_cldy; - qsub3[la][jclea - 1] = qgcm3[la] * mass_part_fac_clea; - qsub3[la][jcldy - 1] = qgcm3[la] * mass_part_fac_cldy; - } - } - - // ************************************************************************************************* - // Category II: all clear, or cld < 1e-5 - // In this case, zfclea=1 and zfcldy=0 - // ************************************************************************************************* - } else if ((jclea == 1) && (jcldy == 0) && (nsubarea == 1)) { - // - // put all the gases and interstitial aerosols in the clear sub-area - // and set mix-ratios = 0 in cloudy sub-area - // for cloud-borne aerosol, do nothing - // because the grid-cell-mean cloud-borne aerosol will be left - // unchanged (i.e., this routine only changes qqcw when cld >= 1e-5) - // - - for (int lmz = 0; lmz < gas_pcnst(); ++lmz) { - if (0 < lmapcc_all(lmz)) { - qsub1[lmz][jclea - 1] = qgcm1[lmz]; - qsub2[lmz][jclea - 1] = qgcm2[lmz]; - qsub3[lmz][jclea - 1] = qgcm3[lmz]; - qqcwsub2[lmz][jclea - 1] = qqcwgcm2[lmz]; - qqcwsub3[lmz][jclea - 1] = qqcwgcm3[lmz]; - } - } - // ************************************************************************************************* - // Category III: all cloudy, or cld > 0.999 - // in this case, zfcldy= and zfclea=0 - // ************************************************************************************************* - } else if ((jclea == 0) && (jcldy == 1) && (nsubarea == 1)) { - // - // put all the gases and interstitial aerosols in the cloudy sub-area - // and set mix-ratios = 0 in clear sub-area - // - for (int lmz = 0; lmz < gas_pcnst(); ++lmz) { - if (0 < lmapcc_all(lmz)) { - qsub1[lmz][jcldy - 1] = qgcm1[lmz]; - qsub2[lmz][jcldy - 1] = qgcm2[lmz]; - qsub3[lmz][jcldy - 1] = qgcm3[lmz]; - qqcwsub2[lmz][jcldy - 1] = qqcwgcm2[lmz]; - qqcwsub3[lmz][jcldy - 1] = qqcwgcm3[lmz]; - } - } - // ************************************************************************************************* - } else { // this should not happen - EKAT_KERNEL_REQUIRE_MSG(false, "*** modal_aero_amicphys - bad jclea, jcldy, nsubarea!"); - } - // ************************************************************************************************* - - // ------------------------------------------------------------------------------------ - // aerosol water -- how to treat this in sub-areas needs more work/thinking - // currently modal_aero_water_uptake calculates qaerwat using - // the grid-cell mean interstital-aerosol mix-rats and the clear-area rh - for (int jsub = 0; jsub < nsubarea; ++jsub) - for (int i = 0; i < num_modes; ++i) - qaerwatsub3[i][jsub] = qaerwatgcm3[i]; - - // ------------------------------------------------------------------------------------ - if (nsubarea == 1) { - // the j=1 subarea is used for some diagnostics - // but is not used in actual calculations - const int j = 1; - for (int i = 0; i < gas_pcnst(); ++i) { - qsub1[i][j] = 0.0; - qsub2[i][j] = 0.0; - qsub3[i][j] = 0.0; - qqcwsub2[i][j] = 0.0; - qqcwsub3[i][j] = 0.0; - } - } -} - -KOKKOS_INLINE_FUNCTION -void mam_amicphys_1subarea_clear( - const AmicPhysConfig& config, const int nstep, const Real deltat, const int jsub, - const int nsubarea, const bool iscldy_subarea, const Real afracsub, - const Real temp, const Real pmid, const Real pdel, const Real zmid, - const Real pblh, const Real relhum, Real dgn_a[AeroConfig::num_modes()], - Real dgn_awet[AeroConfig::num_modes()], - Real wetdens[AeroConfig::num_modes()], - const Real qgas1[AeroConfig::num_gas_ids()], - const Real qgas3[AeroConfig::num_gas_ids()], - Real qgas4[AeroConfig::num_gas_ids()], - Real qgas_delaa[AeroConfig::num_gas_ids()][nqtendaa()], - const Real qnum3[AeroConfig::num_modes()], - Real qnum4[AeroConfig::num_modes()], - Real qnum_delaa[AeroConfig::num_modes()][nqtendaa()], - const Real qaer3[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()], - Real qaer4[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()], - Real qaer_delaa[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()] - [nqtendaa()], - const Real qwtr3[AeroConfig::num_modes()], - Real qwtr4[AeroConfig::num_modes()]) { - static constexpr int num_gas_ids = AeroConfig::num_gas_ids(); - static constexpr int num_modes = AeroConfig::num_modes(); - static constexpr int num_aerosol_ids = AeroConfig::num_aerosol_ids(); - - static constexpr int igas_h2so4 = static_cast(GasId::H2SO4); - // Turn off nh3 for now. This is a future enhancement. - static constexpr int igas_nh3 = -999888777; // Same as mam_refactor - static constexpr int iaer_so4 = static_cast(AeroId::SO4); - static constexpr int iaer_pom = static_cast(AeroId::POM); - - const AeroId gas_to_aer[num_gas_ids] = {AeroId::SOA, AeroId::SO4, - AeroId::None}; - - const bool l_gas_condense_to_mode[num_gas_ids][num_modes] = { - {true, true, true, true}, - {true, true, true, true}, - {false, false, false, false}}; - enum { NA, ANAL, IMPL }; - const int eqn_and_numerics_category[num_gas_ids] = {IMPL, ANAL, ANAL}; - - // air molar density (kmol/m3) - // const Real r_universal = Constants::r_gas; // [mJ/(K mol)] - const Real r_universal = 8.314467591; // [mJ/(mol)] as in mam_refactor - const Real aircon = pmid / (1000 * r_universal * temp); - const Real alnsg_aer[num_modes] = {0.58778666490211906, 0.47000362924573563, - 0.58778666490211906, 0.47000362924573563}; - const Real uptk_rate_factor[num_gas_ids] = {0.81, 1.0, 1.0}; - // calculates changes to gas and aerosol sub-area TMRs (tracer mixing ratios) - // qgas3, qaer3, qnum3 are the current incoming TMRs - // qgas4, qaer4, qnum4 are the updated outgoing TMRs - // - // this routine calculates changes involving - // gas-aerosol exchange (condensation/evaporation) - // growth from smaller to larger modes (renaming) due to condensation - // new particle nucleation - // coagulation - // transfer of particles from hydrophobic modes to hydrophilic modes - // (aging) - // due to condensation and coagulation - // - // qXXXN (X=gas,aer,wat,num; N=1:4) are sub-area mixing ratios - // XXX=gas - gas species - // XXX=aer - aerosol mass species (excluding water) - // XXX=wat - aerosol water - // XXX=num - aerosol number - // N=1 - before gas-phase chemistry - // N=2 - before cloud chemistry - // N=3 - current incoming values (before gas-aerosol exchange, newnuc, - // coag) N=4 - updated outgoing values (after gas-aerosol exchange, - // newnuc, coag) - // - // qXXX_delaa are TMR changes (not tendencies) - // for different processes, which are used to produce history output - // for a clear sub-area, the processes are condensation/evaporation (and - // associated aging), renaming, coagulation, and nucleation - - Real qgas_cur[num_gas_ids]; - for (int i = 0; i < num_gas_ids; ++i) - qgas_cur[i] = qgas3[i]; - Real qaer_cur[num_aerosol_ids][num_modes]; - for (int i = 0; i < num_aerosol_ids; ++i) - for (int j = 0; j < num_modes; ++j) - qaer_cur[i][j] = qaer3[i][j]; - - Real qnum_cur[num_modes]; - for (int j = 0; j < num_modes; ++j) - qnum_cur[j] = qnum3[j]; - Real qwtr_cur[num_modes]; - for (int j = 0; j < num_modes; ++j) - qwtr_cur[j] = qwtr3[j]; - - // qgas_netprod_otrproc = gas net production rate from other processes - // such as gas-phase chemistry and emissions (mol/mol/s) - // this allows the condensation (gasaerexch) routine to apply production and - // condensation loss - // together, which is more accurate numerically - // NOTE - must be >= zero, as numerical method can fail when it is negative - // NOTE - currently only the values for h2so4 and nh3 should be non-zero - Real qgas_netprod_otrproc[num_gas_ids] = {}; - if (config.do_cond && config.gaexch_h2so4_uptake_optaa == 2) { - for (int igas = 0; igas < num_gas_ids; ++igas) { - if (igas == igas_h2so4 || igas == igas_nh3) { - // if config.gaexch_h2so4_uptake_optaa == 2, then - // if qgas increases from pre-gaschem to post-cldchem, - // start from the pre-gaschem mix-ratio and add in the production - // during the integration - // if it decreases, - // start from post-cldchem mix-ratio - // *** currently just do this for h2so4 and nh3 - qgas_netprod_otrproc[igas] = (qgas3[igas] - qgas1[igas]) / deltat; - if (qgas_netprod_otrproc[igas] >= 0.0) - qgas_cur[igas] = qgas1[igas]; - else - qgas_netprod_otrproc[igas] = 0.0; - } - } - } - Real qgas_del_cond[num_gas_ids] = {}; - Real qgas_del_nnuc[num_gas_ids] = {}; - Real qgas_del_cond_only[num_gas_ids] = {}; - Real qaer_del_cond[num_aerosol_ids][num_modes] = {}; - Real qaer_del_rnam[num_aerosol_ids][num_modes] = {}; - Real qaer_del_nnuc[num_aerosol_ids][num_modes] = {}; - Real qaer_del_coag[num_aerosol_ids][num_modes] = {}; - Real qaer_delsub_coag_in[num_aerosol_ids][AeroConfig::max_agepair()] = {}; - Real qaer_delsub_cond[num_aerosol_ids][num_modes] = {}; - Real qaer_delsub_coag[num_aerosol_ids][num_modes] = {}; - Real qaer_del_cond_only[num_aerosol_ids][num_modes] = {}; - Real qnum_del_cond[num_modes] = {}; - Real qnum_del_rnam[num_modes] = {}; - Real qnum_del_nnuc[num_modes] = {}; - Real qnum_del_coag[num_modes] = {}; - Real qnum_delsub_cond[num_modes] = {}; - Real qnum_delsub_coag[num_modes] = {}; - Real qnum_del_cond_only[num_modes] = {}; - Real dnclusterdt = 0.0; - - const int ntsubstep = 1; - Real dtsubstep = deltat; - if (ntsubstep > 1) - dtsubstep = deltat / ntsubstep; - Real del_h2so4_gasprod = - haero::max(qgas3[igas_h2so4] - qgas1[igas_h2so4], 0.0) / ntsubstep; - - // loop over multiple time sub-steps - for (int jtsubstep = 1; jtsubstep <= ntsubstep; ++jtsubstep) { - // gas-aerosol exchange - Real uptkrate_h2so4 = 0.0; - Real del_h2so4_aeruptk = 0.0; - Real qaer_delsub_grow4rnam[num_aerosol_ids][num_modes] = {}; - Real qgas_avg[num_gas_ids] = {}; - Real qnum_sv1[num_modes] = {}; - Real qaer_sv1[num_aerosol_ids][num_modes] = {}; - Real qgas_sv1[num_gas_ids] = {}; - - if (config.do_cond) { - - const bool l_calc_gas_uptake_coeff = jtsubstep == 1; - Real uptkaer[num_gas_ids][num_modes] = {}; - - for (int i = 0; i < num_gas_ids; ++i) - qgas_sv1[i] = qgas_cur[i]; - for (int i = 0; i < num_modes; ++i) - qnum_sv1[i] = qnum_cur[i]; - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) - qaer_sv1[j][i] = qaer_cur[j][i]; - - // time sub-step - const Real dtsub_soa_fixed = -1.0; - // Integration order - const int nghq = 2; - const int ntot_soamode = 4; - int niter_out = 0; - Real g0_soa_out = 0; - gasaerexch::mam_gasaerexch_1subarea( - nghq, igas_h2so4, igas_nh3, ntot_soamode, gas_to_aer, iaer_so4, - iaer_pom, l_calc_gas_uptake_coeff, l_gas_condense_to_mode, - eqn_and_numerics_category, dtsubstep, dtsub_soa_fixed, temp, pmid, - aircon, num_gas_ids, qgas_cur, qgas_avg, qgas_netprod_otrproc, - qaer_cur, qnum_cur, dgn_awet, alnsg_aer, uptk_rate_factor, uptkaer, - uptkrate_h2so4, niter_out, g0_soa_out); - - if (config.newnuc_h2so4_conc_optaa == 11) - qgas_avg[igas_h2so4] = - 0.5 * (qgas_sv1[igas_h2so4] + qgas_cur[igas_h2so4]); - else if (config.newnuc_h2so4_conc_optaa == 12) - qgas_avg[igas_h2so4] = qgas_cur[igas_h2so4]; - - for (int i = 0; i < num_gas_ids; ++i) - qgas_del_cond[i] += - (qgas_cur[i] - (qgas_sv1[i] + qgas_netprod_otrproc[i] * dtsubstep)); - - for (int i = 0; i < num_modes; ++i) - qnum_delsub_cond[i] = qnum_cur[i] - qnum_sv1[i]; - for (int i = 0; i < num_aerosol_ids; ++i) - for (int j = 0; j < num_modes; ++j) - qaer_delsub_cond[i][j] = qaer_cur[i][j] - qaer_sv1[i][j]; - - // qaer_del_grow4rnam = change in qaer_del_cond during latest condensation - // calculations - for (int i = 0; i < num_aerosol_ids; ++i) - for (int j = 0; j < num_modes; ++j) - qaer_delsub_grow4rnam[i][j] = qaer_cur[i][j] - qaer_sv1[i][j]; - for (int i = 0; i < num_gas_ids; ++i) - qgas_del_cond_only[i] = qgas_del_cond[i]; - for (int i = 0; i < num_aerosol_ids; ++i) - for (int j = 0; j < num_modes; ++j) - qaer_del_cond_only[i][j] = qaer_delsub_cond[i][j]; - for (int i = 0; i < num_modes; ++i) - qnum_del_cond_only[i] = qnum_delsub_cond[i]; - del_h2so4_aeruptk = - qgas_cur[igas_h2so4] - - (qgas_sv1[igas_h2so4] + qgas_netprod_otrproc[igas_h2so4] * dtsubstep); - } else { - for (int i = 0; i < num_gas_ids; ++i) - qgas_avg[i] = qgas_cur[i]; - } - - // renaming after "continuous growth" - if (config.do_rename) { - constexpr int nmodes = AeroConfig::num_modes(); - constexpr int naerosol_species = AeroConfig::num_aerosol_ids(); - const Real smallest_dryvol_value = 1.0e-25; // BAD_CONSTANT - const int dest_mode_of_mode[nmodes] = {-1, 0, -1, -1}; - - Real qnumcw_cur[num_modes] = {}; - Real qaercw_cur[num_aerosol_ids][num_modes] = {}; - Real qaercw_delsub_grow4rnam[num_aerosol_ids][num_modes] = {}; - Real mean_std_dev[nmodes]; - Real fmode_dist_tail_fac[nmodes]; - Real v2n_lo_rlx[nmodes]; - Real v2n_hi_rlx[nmodes]; - Real ln_diameter_tail_fac[nmodes]; - int num_pairs = 0; - Real diameter_cutoff[nmodes]; - Real ln_dia_cutoff[nmodes]; - Real diameter_threshold[nmodes]; - Real mass_2_vol[naerosol_species] = {0.15, - 6.4971751412429377e-002, - 0.15, - 7.0588235294117650e-003, - 3.0789473684210526e-002, - 5.1923076923076926e-002, - 156.20986883198000}; - - rename::find_renaming_pairs(dest_mode_of_mode, // in - mean_std_dev, // out - fmode_dist_tail_fac, // out - v2n_lo_rlx, // out - v2n_hi_rlx, // out - ln_diameter_tail_fac, // out - num_pairs, // out - diameter_cutoff, // out - ln_dia_cutoff, diameter_threshold); - - for (int i = 0; i < num_modes; ++i) - qnum_sv1[i] = qnum_cur[i]; - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) - qaer_sv1[j][i] = qaer_cur[j][i]; - Real dgnum_amode[nmodes]; - for (int m = 0; m < nmodes; ++m) { - dgnum_amode[m] = modes(m).nom_diameter; - } - - { - Real qmol_i_cur[num_modes][num_aerosol_ids]; - Real qmol_i_del[num_modes][num_aerosol_ids]; - Real qmol_c_cur[num_modes][num_aerosol_ids]; - Real qmol_c_del[num_modes][num_aerosol_ids]; - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) { - qmol_i_cur[i][j] = qaer_cur[j][i]; - qmol_i_del[i][j] = qaer_delsub_grow4rnam[j][i]; - qmol_c_cur[i][j] = qaercw_cur[j][i]; - qmol_c_del[i][j] = qaercw_delsub_grow4rnam[j][i]; - } - Rename rename; - rename.mam_rename_1subarea_( - iscldy_subarea, smallest_dryvol_value, dest_mode_of_mode, - mean_std_dev, fmode_dist_tail_fac, v2n_lo_rlx, v2n_hi_rlx, - ln_diameter_tail_fac, num_pairs, diameter_cutoff, ln_dia_cutoff, - diameter_threshold, mass_2_vol, dgnum_amode, qnum_cur, qmol_i_cur, - qmol_i_del, qnumcw_cur, qmol_c_cur, qmol_c_del); - - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) { - qaer_cur[j][i] = qmol_i_cur[i][j]; - qaer_delsub_grow4rnam[j][i] = qmol_i_del[i][j]; - qaercw_cur[j][i] = qmol_c_cur[i][j]; - qaercw_delsub_grow4rnam[j][i] = qmol_c_del[i][j]; - } - } - - for (int i = 0; i < num_modes; ++i) - qnum_del_rnam[i] += qnum_cur[i] - qnum_sv1[i]; - for (int i = 0; i < num_aerosol_ids; ++i) - for (int j = 0; j < num_modes; ++j) - qaer_del_rnam[i][j] += qaer_cur[i][j] - qaer_sv1[i][j]; - } - - // new particle formation (nucleation) - if (config.do_newnuc) { - for (int i = 0; i < num_gas_ids; ++i) - qgas_sv1[i] = qgas_cur[i]; - for (int i = 0; i < num_modes; ++i) - qnum_sv1[i] = qnum_cur[i]; - Real qaer_cur_tmp[num_modes][num_aerosol_ids]; - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) { - qaer_sv1[j][i] = qaer_cur[j][i]; - qaer_cur_tmp[i][j] = qaer_cur[j][i]; - } - Real dnclusterdt_substep = 0; - Real dndt_ait = 0; - Real dmdt_ait = 0; - Real dso4dt_ait = 0; - Real dnh4dt_ait = 0; - Nucleation nucleation; - Nucleation::Config config; - config.dens_so4a_host = 1770; - config.mw_nh4a_host = 115; - config.mw_so4a_host = 115; - config.accom_coef_h2so4 = 0.65; - AeroConfig aero_config; - nucleation.init(aero_config, config); - nucleation.compute_tendencies_( - dtsubstep, temp, pmid, aircon, zmid, pblh, relhum, uptkrate_h2so4, - del_h2so4_gasprod, del_h2so4_aeruptk, qgas_cur, qgas_avg, qnum_cur, - qaer_cur_tmp, qwtr_cur, dndt_ait, dmdt_ait, dso4dt_ait, dnh4dt_ait, - dnclusterdt_substep); - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) - qaer_cur[j][i] = qaer_cur_tmp[i][j]; - - //! Apply the tendencies to the prognostics. - const int nait = static_cast(ModeIndex::Aitken); - qnum_cur[nait] += dndt_ait * dtsubstep; - - if (dso4dt_ait > 0.0) { - static constexpr int iaer_so4 = static_cast(AeroId::SO4); - static constexpr int igas_h2so4 = static_cast(GasId::H2SO4); - - Real delta_q = dso4dt_ait * dtsubstep; - qaer_cur[iaer_so4][nait] += delta_q; - delta_q = haero::min(delta_q, qgas_cur[igas_h2so4]); - qgas_cur[igas_h2so4] -= delta_q; - } - - if (igas_nh3 > 0 && dnh4dt_ait > 0.0) { - static constexpr int iaer_nh4 = - -9999999; // static_cast(AeroId::NH4); - - Real delta_q = dnh4dt_ait * dtsubstep; - qaer_cur[iaer_nh4][nait] += delta_q; - delta_q = haero::min(delta_q, qgas_cur[igas_nh3]); - qgas_cur[igas_nh3] -= delta_q; - } - for (int i = 0; i < num_gas_ids; ++i) - qgas_del_nnuc[i] += (qgas_cur[i] - qgas_sv1[i]); - for (int i = 0; i < num_modes; ++i) - qnum_del_nnuc[i] += (qnum_cur[i] - qnum_sv1[i]); - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) - qaer_del_nnuc[j][i] += (qaer_cur[j][i] - qaer_sv1[j][i]); - - dnclusterdt = dnclusterdt + dnclusterdt_substep * (dtsubstep / deltat); - } - - // coagulation part - if (config.do_coag) { - for (int i = 0; i < num_modes; ++i) - qnum_sv1[i] = qnum_cur[i]; - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) - qaer_sv1[j][i] = qaer_cur[j][i]; - coagulation::mam_coag_1subarea(dtsubstep, temp, pmid, aircon, dgn_a, - dgn_awet, wetdens, qnum_cur, qaer_cur, - qaer_delsub_coag_in); - for (int i = 0; i < num_modes; ++i) - qnum_delsub_coag[i] = qnum_cur[i] - qnum_sv1[i]; - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) - qaer_delsub_coag[j][i] = qaer_cur[j][i] - qaer_sv1[j][i]; - } - - // primary carbon aging - - aging::mam_pcarbon_aging_1subarea( - dgn_a, qnum_cur, qnum_delsub_cond, qnum_delsub_coag, qaer_cur, - qaer_delsub_cond, qaer_delsub_coag, qaer_delsub_coag_in); - - // accumulate sub-step q-dels - if (config.do_coag) { - for (int i = 0; i < num_modes; ++i) - qnum_del_coag[i] += qnum_delsub_coag[i]; - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) - qaer_del_coag[j][i] += qaer_delsub_coag[j][i]; - } - if (config.do_cond) { - for (int i = 0; i < num_modes; ++i) - qnum_del_cond[i] += qnum_delsub_cond[i]; - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) - qaer_del_cond[j][i] += qaer_delsub_cond[j][i]; - } - } - - // final mix ratios - for (int i = 0; i < num_gas_ids; ++i) - qgas4[i] = qgas_cur[i]; - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) - qaer4[j][i] = qaer_cur[j][i]; - for (int i = 0; i < num_modes; ++i) - qnum4[i] = qnum_cur[i]; - for (int i = 0; i < num_modes; ++i) - qwtr4[i] = qwtr_cur[i]; - - // final mix ratio changes - for (int i = 0; i < num_gas_ids; ++i) { - qgas_delaa[i][iqtend_cond()] = qgas_del_cond[i]; - qgas_delaa[i][iqtend_rnam()] = 0.0; - qgas_delaa[i][iqtend_nnuc()] = qgas_del_nnuc[i]; - qgas_delaa[i][iqtend_coag()] = 0.0; - qgas_delaa[i][iqtend_cond_only()] = qgas_del_cond_only[i]; - } - for (int i = 0; i < num_modes; ++i) { - qnum_delaa[i][iqtend_cond()] = qnum_del_cond[i]; - qnum_delaa[i][iqtend_rnam()] = qnum_del_rnam[i]; - qnum_delaa[i][iqtend_nnuc()] = qnum_del_nnuc[i]; - qnum_delaa[i][iqtend_coag()] = qnum_del_coag[i]; - qnum_delaa[i][iqtend_cond_only()] = qnum_del_cond_only[i]; - } - for (int j = 0; j < num_aerosol_ids; ++j) { - for (int i = 0; i < num_modes; ++i) { - qaer_delaa[j][i][iqtend_cond()] = qaer_del_cond[j][i]; - qaer_delaa[j][i][iqtend_rnam()] = qaer_del_rnam[j][i]; - qaer_delaa[j][i][iqtend_nnuc()] = qaer_del_nnuc[j][i]; - qaer_delaa[j][i][iqtend_coag()] = qaer_del_coag[j][i]; - qaer_delaa[j][i][iqtend_cond_only()] = qaer_del_cond_only[j][i]; - } - } -} - -KOKKOS_INLINE_FUNCTION -void mam_amicphys_1subarea_cloudy( - const AmicPhysConfig& config, const int nstep, const Real deltat, const int jsub, - const int nsubarea, const bool iscldy_subarea, const Real afracsub, - const Real temp, const Real pmid, const Real pdel, const Real zmid, - const Real pblh, const Real relhum, Real dgn_a[AeroConfig::num_modes()], - Real dgn_awet[AeroConfig::num_modes()], - Real wetdens[AeroConfig::num_modes()], - const Real qgas1[AeroConfig::num_gas_ids()], - const Real qgas3[AeroConfig::num_gas_ids()], - Real qgas4[AeroConfig::num_gas_ids()], - Real qgas_delaa[AeroConfig::num_gas_ids()][nqtendaa()], - const Real qnum3[AeroConfig::num_modes()], - Real qnum4[AeroConfig::num_modes()], - Real qnum_delaa[AeroConfig::num_modes()][nqtendaa()], - const Real qaer2[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()], - const Real qaer3[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()], - Real qaer4[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()], - Real qaer_delaa[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()] - [nqtendaa()], - const Real qwtr3[AeroConfig::num_modes()], - Real qwtr4[AeroConfig::num_modes()], - const Real qnumcw3[AeroConfig::num_modes()], - Real qnumcw4[AeroConfig::num_modes()], - Real qnumcw_delaa[AeroConfig::num_modes()][nqqcwtendaa()], - const Real qaercw2[AeroConfig::num_gas_ids()][AeroConfig::num_modes()], - const Real qaercw3[AeroConfig::num_gas_ids()][AeroConfig::num_modes()], - Real qaercw4[AeroConfig::num_gas_ids()][AeroConfig::num_modes()], - Real qaercw_delaa[AeroConfig::num_gas_ids()][AeroConfig::num_modes()] - [nqqcwtendaa()]) { - - // - // calculates changes to gas and aerosol sub-area TMRs (tracer mixing ratios) - // qgas3, qaer3, qaercw3, qnum3, qnumcw3 are the current incoming TMRs - // qgas4, qaer4, qaercw4, qnum4, qnumcw4 are the updated outgoing TMRs - // - // when config.do_cond = false, this routine only calculates changes involving - // growth from smaller to larger modes (renaming) following cloud chemistry - // so gas TMRs are not changed - // when config.do_cond = true, this routine also calculates changes involving - // gas-aerosol exchange (condensation/evaporation) - // transfer of particles from hydrophobic modes to hydrophilic modes - // (aging) - // due to condensation - // currently this routine does not do - // new particle nucleation - because h2so4 gas conc. should be very low in - // cloudy air coagulation - because cloud-borne aerosol would need to be - // included - // - - // qXXXN (X=gas,aer,wat,num; N=1:4) are sub-area mixing ratios - // XXX=gas - gas species - // XXX=aer - aerosol mass species (excluding water) - // XXX=wat - aerosol water - // XXX=num - aerosol number - // N=1 - before gas-phase chemistry - // N=2 - before cloud chemistry - // N=3 - current incoming values (before gas-aerosol exchange, newnuc, - // coag) N=4 - updated outgoing values (after gas-aerosol exchange, - // newnuc, coag) - // - // qXXX_delaa are TMR changes (not tendencies) - // for different processes, which are used to produce history output - // for a clear sub-area, the processes are condensation/evaporation (and - // associated aging), - // renaming, coagulation, and nucleation - - // qxxx_del_yyyy are mix-ratio changes over full time step (deltat) - // qxxx_delsub_yyyy are mix-ratio changes over time sub-step (dtsubstep) - - static constexpr int num_gas_ids = AeroConfig::num_gas_ids(); - static constexpr int num_modes = AeroConfig::num_modes(); - static constexpr int num_aerosol_ids = AeroConfig::num_aerosol_ids(); - - static constexpr int igas_h2so4 = static_cast(GasId::H2SO4); - // Turn off nh3 for now. This is a future enhancement. - static constexpr int igas_nh3 = -999888777; // Same as mam_refactor - static constexpr int iaer_so4 = static_cast(AeroId::SO4); - static constexpr int iaer_pom = static_cast(AeroId::POM); - - const AeroId gas_to_aer[num_gas_ids] = {AeroId::SOA, AeroId::SO4, - AeroId::None}; - const bool l_gas_condense_to_mode[num_gas_ids][num_modes] = { - {true, true, true, true}, - {true, true, true, true}, - {false, false, false, false}}; - enum { NA, ANAL, IMPL }; - const int eqn_and_numerics_category[num_gas_ids] = {IMPL, ANAL, ANAL}; - // air molar density (kmol/m3) - // In order to try to match the results in mam_refactor - // set r_universal as [mJ/(mol)] as in mam_refactor. - // const Real r_universal = Constants::r_gas; // [mJ/(K mol)] - const Real r_universal = 8.314467591; // [mJ/(mol)] as in mam_refactor - const Real aircon = pmid / (1000 * r_universal * temp); - const Real alnsg_aer[num_modes] = {0.58778666490211906, 0.47000362924573563, - 0.58778666490211906, 0.47000362924573563}; - const Real uptk_rate_factor[num_gas_ids] = {0.81, 1.0, 1.0}; - - Real qgas_cur[num_gas_ids]; - for (int i = 0; i < num_gas_ids; ++i) - qgas_cur[i] = qgas3[i]; - Real qaer_cur[num_aerosol_ids][num_modes]; - for (int i = 0; i < num_aerosol_ids; ++i) - for (int j = 0; j < num_modes; ++j) - qaer_cur[i][j] = qaer3[i][j]; - - Real qnum_cur[num_modes]; - for (int j = 0; j < num_modes; ++j) - qnum_cur[j] = qnum3[j]; - Real qwtr_cur[num_modes]; - for (int j = 0; j < num_modes; ++j) - qwtr_cur[j] = qwtr3[j]; - - Real qnumcw_cur[num_modes]; - for (int j = 0; j < num_modes; ++j) - qnumcw_cur[j] = qnumcw3[j]; - - Real qaercw_cur[num_gas_ids][num_modes]; - for (int i = 0; i < num_gas_ids; ++i) - for (int j = 0; j < num_modes; ++j) - qaercw_cur[i][j] = qaercw3[i][j]; - - Real qgas_netprod_otrproc[num_gas_ids] = {}; - if (config.do_cond && config.gaexch_h2so4_uptake_optaa == 2) { - for (int igas = 0; igas < num_gas_ids; ++igas) { - if (igas == igas_h2so4 || igas == igas_nh3) { - // if gaexch_h2so4_uptake_optaa == 2, then - // if qgas increases from pre-gaschem to post-cldchem, - // start from the pre-gaschem mix-ratio and add in the production - // during the integration - // if it decreases, - // start from post-cldchem mix-ratio - // *** currently just do this for h2so4 and nh3 - qgas_netprod_otrproc[igas] = (qgas3[igas] - qgas1[igas]) / deltat; - if (qgas_netprod_otrproc[igas] >= 0.0) - qgas_cur[igas] = qgas1[igas]; - else - qgas_netprod_otrproc[igas] = 0.0; - } - } - } - Real qgas_del_cond[num_gas_ids] = {}; - Real qgas_del_nnuc[num_gas_ids] = {}; - Real qgas_del_cond_only[num_gas_ids] = {}; - Real qaer_del_cond[num_aerosol_ids][num_modes] = {}; - Real qaer_del_rnam[num_aerosol_ids][num_modes] = {}; - Real qaer_del_nnuc[num_aerosol_ids][num_modes] = {}; - Real qaer_del_coag[num_aerosol_ids][num_modes] = {}; - Real qaer_delsub_cond[num_aerosol_ids][num_modes] = {}; - Real qaer_delsub_coag[num_aerosol_ids][num_modes] = {}; - Real qaer_del_cond_only[num_aerosol_ids][num_modes] = {}; - Real qaercw_del_rnam[num_aerosol_ids][num_modes] = {}; - Real qnum_del_cond[num_modes] = {}; - Real qnum_del_rnam[num_modes] = {}; - Real qnum_del_nnuc[num_modes] = {}; - Real qnum_del_coag[num_modes] = {}; - Real qnum_delsub_cond[num_modes] = {}; - Real qnum_delsub_coag[num_modes] = {}; - Real qnum_del_cond_only[num_modes] = {}; - Real qnumcw_del_rnam[num_modes] = {}; - Real qaer_delsub_coag_in[num_aerosol_ids][AeroConfig::max_agepair()] = {}; - - const int ntsubstep = 1; - Real dtsubstep = deltat; - if (ntsubstep > 1) - dtsubstep = deltat / ntsubstep; - - // loop over multiple time sub-steps - - for (int jtsubstep = 1; jtsubstep <= ntsubstep; ++jtsubstep) { - // gas-aerosol exchange - Real uptkrate_h2so4 = 0.0; - Real qgas_avg[num_gas_ids] = {}; - Real qgas_sv1[num_gas_ids] = {}; - Real qnum_sv1[num_modes] = {}; - Real qaer_sv1[num_aerosol_ids][num_modes] = {}; - Real qaer_delsub_grow4rnam[num_aerosol_ids][num_modes] = {}; - - if (config.do_cond) { - - const bool l_calc_gas_uptake_coeff = jtsubstep == 1; - Real uptkaer[num_gas_ids][num_modes] = {}; - - for (int i = 0; i < num_gas_ids; ++i) - qgas_sv1[i] = qgas_cur[i]; - for (int i = 0; i < num_modes; ++i) - qnum_sv1[i] = qnum_cur[i]; - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) - qaer_sv1[j][i] = qaer_cur[j][i]; - - const int nghq = 2; - const int ntot_soamode = 4; - int niter_out = 0; - Real g0_soa_out = 0; - // time sub-step - const Real dtsub_soa_fixed = -1.0; - gasaerexch::mam_gasaerexch_1subarea( - nghq, igas_h2so4, igas_nh3, ntot_soamode, gas_to_aer, iaer_so4, - iaer_pom, l_calc_gas_uptake_coeff, l_gas_condense_to_mode, - eqn_and_numerics_category, dtsubstep, dtsub_soa_fixed, temp, pmid, - aircon, num_gas_ids, qgas_cur, qgas_avg, qgas_netprod_otrproc, - qaer_cur, qnum_cur, dgn_awet, alnsg_aer, uptk_rate_factor, uptkaer, - uptkrate_h2so4, niter_out, g0_soa_out); - - if (config.newnuc_h2so4_conc_optaa == 11) - qgas_avg[igas_h2so4] = - 0.5 * (qgas_sv1[igas_h2so4] + qgas_cur[igas_h2so4]); - else if (config.newnuc_h2so4_conc_optaa == 12) - qgas_avg[igas_h2so4] = qgas_cur[igas_h2so4]; - - for (int i = 0; i < num_gas_ids; ++i) - qgas_del_cond[i] += - (qgas_cur[i] - (qgas_sv1[i] + qgas_netprod_otrproc[i] * dtsubstep)); - - for (int i = 0; i < num_modes; ++i) - qnum_delsub_cond[i] = qnum_cur[i] - qnum_sv1[i]; - for (int i = 0; i < num_aerosol_ids; ++i) - for (int j = 0; j < num_modes; ++j) - qaer_delsub_cond[i][j] = qaer_cur[i][j] - qaer_sv1[i][j]; - - // qaer_del_grow4rnam = change in qaer_del_cond during latest condensation - // calculations - for (int i = 0; i < num_aerosol_ids; ++i) - for (int j = 0; j < num_modes; ++j) - qaer_delsub_grow4rnam[i][j] = qaer_cur[i][j] - qaer_sv1[i][j]; - for (int i = 0; i < num_gas_ids; ++i) - qgas_del_cond_only[i] = qgas_del_cond[i]; - for (int i = 0; i < num_aerosol_ids; ++i) - for (int j = 0; j < num_modes; ++j) - qaer_del_cond_only[i][j] = qaer_delsub_cond[i][j]; - for (int i = 0; i < num_modes; ++i) - qnum_del_cond_only[i] = qnum_delsub_cond[i]; - - } else { - for (int i = 0; i < num_gas_ids; ++i) - qgas_avg[i] = qgas_cur[i]; - } - // renaming after "continuous growth" - if (config.do_rename) { - constexpr int nmodes = AeroConfig::num_modes(); - constexpr int naerosol_species = AeroConfig::num_aerosol_ids(); - const Real smallest_dryvol_value = 1.0e-25; // BAD_CONSTANT - const int dest_mode_of_mode[nmodes] = {-1, 0, -1, -1}; - - Real qnumcw_cur[num_modes] = {}; - Real qaercw_cur[num_aerosol_ids][num_modes] = {}; - Real qaercw_delsub_grow4rnam[num_aerosol_ids][num_modes] = {}; - Real mean_std_dev[nmodes]; - Real fmode_dist_tail_fac[nmodes]; - Real v2n_lo_rlx[nmodes]; - Real v2n_hi_rlx[nmodes]; - Real ln_diameter_tail_fac[nmodes]; - int num_pairs = 0; - Real diameter_cutoff[nmodes]; - Real ln_dia_cutoff[nmodes]; - Real diameter_threshold[nmodes]; - Real mass_2_vol[naerosol_species] = {0.15, - 6.4971751412429377e-002, - 0.15, - 7.0588235294117650e-003, - 3.0789473684210526e-002, - 5.1923076923076926e-002, - 156.20986883198000}; - - rename::find_renaming_pairs(dest_mode_of_mode, // in - mean_std_dev, // out - fmode_dist_tail_fac, // out - v2n_lo_rlx, // out - v2n_hi_rlx, // out - ln_diameter_tail_fac, // out - num_pairs, // out - diameter_cutoff, // out - ln_dia_cutoff, diameter_threshold); - - for (int i = 0; i < num_modes; ++i) - qnum_sv1[i] = qnum_cur[i]; - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) - qaer_sv1[j][i] = qaer_cur[j][i]; - Real dgnum_amode[nmodes]; - for (int m = 0; m < nmodes; ++m) { - dgnum_amode[m] = modes(m).nom_diameter; - } - - // qaercw_delsub_grow4rnam = change in qaercw from cloud chemistry - for (int i = 0; i < num_aerosol_ids; ++i) - for (int j = 0; j < num_modes; ++j) - qaercw_delsub_grow4rnam[i][j] = - (qaercw3[i][j] - qaercw2[i][j]) / ntsubstep; - Real qnumcw_sv1[num_modes]; - for (int i = 0; i < num_modes; ++i) - qnumcw_sv1[i] = qnumcw_cur[i]; - Real qaercw_sv1[num_aerosol_ids][num_modes]; - for (int i = 0; i < num_aerosol_ids; ++i) - for (int j = 0; j < num_modes; ++j) - qaercw_sv1[i][j] = qaercw_cur[i][j]; - - { - Real qmol_i_cur[num_modes][num_aerosol_ids]; - Real qmol_i_del[num_modes][num_aerosol_ids]; - Real qmol_c_cur[num_modes][num_aerosol_ids]; - Real qmol_c_del[num_modes][num_aerosol_ids]; - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) { - qmol_i_cur[i][j] = qaer_cur[j][i]; - qmol_i_del[i][j] = qaer_delsub_grow4rnam[j][i]; - qmol_c_cur[i][j] = qaercw_cur[j][i]; - qmol_c_del[i][j] = qaercw_delsub_grow4rnam[j][i]; - } - - Rename rename; - rename.mam_rename_1subarea_( - iscldy_subarea, smallest_dryvol_value, dest_mode_of_mode, - mean_std_dev, fmode_dist_tail_fac, v2n_lo_rlx, v2n_hi_rlx, - ln_diameter_tail_fac, num_pairs, diameter_cutoff, ln_dia_cutoff, - diameter_threshold, mass_2_vol, dgnum_amode, qnum_cur, qmol_i_cur, - qmol_i_del, qnumcw_cur, qmol_c_cur, qmol_c_del); - - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) { - qaer_cur[j][i] = qmol_i_cur[i][j]; - qaer_delsub_grow4rnam[j][i] = qmol_i_del[i][j]; - qaercw_cur[j][i] = qmol_c_cur[i][j]; - qaercw_delsub_grow4rnam[j][i] = qmol_c_del[i][j]; - } - } - for (int i = 0; i < num_modes; ++i) - qnum_del_rnam[i] += qnum_cur[i] - qnum_sv1[i]; - for (int i = 0; i < num_aerosol_ids; ++i) - for (int j = 0; j < num_modes; ++j) - qaer_del_rnam[i][j] += qaer_cur[i][j] - qaer_sv1[i][j]; - for (int i = 0; i < num_modes; ++i) - qnumcw_del_rnam[i] += qnumcw_cur[i] - qnumcw_sv1[i]; - for (int i = 0; i < num_aerosol_ids; ++i) - for (int j = 0; j < num_modes; ++j) - qaercw_del_rnam[i][j] += qaercw_cur[i][j] - qaercw_sv1[i][j]; - } - - // primary carbon aging - if (config.do_cond) { - aging::mam_pcarbon_aging_1subarea( - dgn_a, qnum_cur, qnum_delsub_cond, qnum_delsub_coag, qaer_cur, - qaer_delsub_cond, qaer_delsub_coag, qaer_delsub_coag_in); - } - // accumulate sub-step q-dels - if (config.do_cond) { - for (int i = 0; i < num_modes; ++i) - qnum_del_cond[i] += qnum_delsub_cond[i]; - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) - qaer_del_cond[j][i] += qaer_delsub_cond[j][i]; - } - } - // final mix ratios - for (int i = 0; i < num_gas_ids; ++i) - qgas4[i] = qgas_cur[i]; - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) - qaer4[j][i] = qaer_cur[j][i]; - for (int i = 0; i < num_modes; ++i) - qnum4[i] = qnum_cur[i]; - for (int i = 0; i < num_modes; ++i) - qwtr4[i] = qwtr_cur[i]; - for (int i = 0; i < num_modes; ++i) - qnumcw4[i] = qnumcw_cur[i]; - for (int i = 0; i < num_gas_ids; ++i) - for (int j = 0; j < num_modes; ++j) - qaercw4[i][j] = qaercw_cur[i][j]; - - // final mix ratio changes - for (int i = 0; i < num_gas_ids; ++i) { - qgas_delaa[i][iqtend_cond()] = qgas_del_cond[i]; - qgas_delaa[i][iqtend_rnam()] = 0.0; - qgas_delaa[i][iqtend_nnuc()] = qgas_del_nnuc[i]; - qgas_delaa[i][iqtend_coag()] = 0.0; - qgas_delaa[i][iqtend_cond_only()] = qgas_del_cond_only[i]; - } - for (int i = 0; i < num_modes; ++i) { - qnum_delaa[i][iqtend_cond()] = qnum_del_cond[i]; - qnum_delaa[i][iqtend_rnam()] = qnum_del_rnam[i]; - qnum_delaa[i][iqtend_nnuc()] = qnum_del_nnuc[i]; - qnum_delaa[i][iqtend_coag()] = qnum_del_coag[i]; - qnum_delaa[i][iqtend_cond_only()] = qnum_del_cond_only[i]; - } - for (int j = 0; j < num_aerosol_ids; ++j) { - for (int i = 0; i < num_modes; ++i) { - qaer_delaa[j][i][iqtend_cond()] = qaer_del_cond[j][i]; - qaer_delaa[j][i][iqtend_rnam()] = qaer_del_rnam[j][i]; - qaer_delaa[j][i][iqtend_nnuc()] = qaer_del_nnuc[j][i]; - qaer_delaa[j][i][iqtend_coag()] = qaer_del_coag[j][i]; - qaer_delaa[j][i][iqtend_cond_only()] = qaer_del_cond_only[j][i]; - } - } - for (int i = 0; i < num_modes; ++i) - qnumcw_delaa[i][iqqcwtend_rnam()] = qnumcw_del_rnam[i]; - for (int i = 0; i < num_aerosol_ids; ++i) - for (int j = 0; j < num_modes; ++j) - qaercw_delaa[i][j][iqqcwtend_rnam()] = qaercw_del_rnam[i][j]; -} - -KOKKOS_INLINE_FUNCTION -void mam_amicphys_1gridcell( - const AmicPhysConfig& config, const int nstep, const Real deltat, const int nsubarea, - const int ncldy_subarea, const bool iscldy_subarea[maxsubarea()], - const Real afracsub[maxsubarea()], const Real temp, const Real pmid, - const Real pdel, const Real zmid, const Real pblh, - const Real relhumsub[maxsubarea()], Real dgn_a[AeroConfig::num_modes()], - Real dgn_awet[AeroConfig::num_modes()], - Real wetdens[AeroConfig::num_modes()], - const Real qsub1[AeroConfig::num_gas_ids()][maxsubarea()], - const Real qsub2[AeroConfig::num_gas_ids()][maxsubarea()], - const Real qqcwsub2[AeroConfig::num_gas_ids()][maxsubarea()], - const Real qsub3[AeroConfig::num_gas_ids()][maxsubarea()], - const Real qqcwsub3[AeroConfig::num_gas_ids()][maxsubarea()], - Real qaerwatsub3[AeroConfig::num_modes()][maxsubarea()], - Real qsub4[AeroConfig::num_gas_ids()][maxsubarea()], - Real qqcwsub4[AeroConfig::num_gas_ids()][maxsubarea()], - Real qaerwatsub4[AeroConfig::num_modes()][maxsubarea()], - Real qsub_tendaa[AeroConfig::num_gas_ids()][nqtendaa()][maxsubarea()], - Real qqcwsub_tendaa[AeroConfig::num_gas_ids()][nqqcwtendaa()][maxsubarea()]) { - - // - // calculates changes to gas and aerosol sub-area TMRs (tracer mixing ratios) - // qsub3 and qqcwsub3 are the incoming current TMRs - // qsub4 and qqcwsub4 are the outgoing updated TMRs - // - // qsubN and qqcwsubN (N=1:4) are tracer mixing ratios (TMRs, mol/mol or - // #/kmol) in sub-areas - // currently there are just clear and cloudy sub-areas - // the N=1:4 have same meanings as for qgcmN - // N=1 - before gas-phase chemistry - // N=2 - before cloud chemistry - // N=3 - incoming values (before gas-aerosol exchange, newnuc, coag) - // N=4 - outgoing values (after gas-aerosol exchange, newnuc, coag) - // qsub_tendaa and qqcwsub_tendaa are TMR tendencies - // for different processes, which are used to produce history output - // the processes are condensation/evaporation (and associated aging), - // renaming, coagulation, and nucleation - - static constexpr int num_gas_ids = AeroConfig::num_gas_ids(); - static constexpr int num_modes = AeroConfig::num_modes(); - static constexpr int num_aerosol_ids = AeroConfig::num_aerosol_ids(); - - // the q--4 values will be equal to q--3 values unless they get changed - for (int i = 0; i < num_gas_ids; ++i) - for (int j = 0; j < maxsubarea(); ++j) { - qsub4[i][j] = qsub3[i][j]; - qqcwsub4[i][j] = qqcwsub3[i][j]; - } - for (int i = 0; i < num_modes; ++i) - for (int j = 0; j < maxsubarea(); ++j) - qaerwatsub4[i][j] = qaerwatsub3[i][j]; - for (int i = 0; i < num_gas_ids; ++i) - for (int j = 0; j < nqtendaa(); ++j) - for (int k = 0; k < maxsubarea(); ++k) - qsub_tendaa[i][j][k] = 0; - for (int i = 0; i < num_gas_ids; ++i) - for (int j = 0; j < nqqcwtendaa(); ++j) - for (int k = 0; k < maxsubarea(); ++k) - qqcwsub_tendaa[i][j][k] = 0.0; - - for (int jsub = 0; jsub < nsubarea; ++jsub) { - AmicPhysConfig sub_config = config; - if (iscldy_subarea[jsub]) { - sub_config.do_cond = config.do_cond; - sub_config.do_rename = config.do_rename; - sub_config.do_newnuc = false; - sub_config.do_coag = false; - } - const bool do_map_gas_sub = sub_config.do_cond || sub_config.do_newnuc; - - // map incoming sub-area mix-ratios to gas/aer/num arrays - Real qgas1[num_gas_ids] = {}; - Real qgas3[num_gas_ids] = {}; - Real qgas4[num_gas_ids] = {}; - if (do_map_gas_sub) { - // for cldy subarea, only do gases if doing gaexch - for (int igas = 0; igas < 2; ++igas) { - const int l = lmap_gas(igas); - qgas1[igas] = qsub1[l][jsub] * fcvt_gas(igas); - qgas3[igas] = qsub3[l][jsub] * fcvt_gas(igas); - qgas4[igas] = qgas3[igas]; - } - } - Real qaer2[num_aerosol_ids][num_modes] = {}; - Real qaer3[num_aerosol_ids][num_modes] = {}; - Real qnum3[num_modes] = {}; - Real qaer4[num_aerosol_ids][num_modes] = {}; - Real qnum4[num_modes] = {}; - Real qwtr3[num_modes] = {}; - Real qwtr4[num_modes] = {}; - for (int n = 0; n < num_modes; ++n) { - qnum3[n] = qsub3[n][jsub] * fcvt_num(); - qnum4[n] = qnum3[n]; - for (int iaer = 0; iaer < num_aerosol_ids; ++iaer) { - qaer2[iaer][n] = qsub2[iaer][jsub] * fcvt_aer(iaer); - qaer3[iaer][n] = qsub3[iaer][jsub] * fcvt_aer(iaer); - qaer4[iaer][n] = qaer3[iaer][n]; - } - qwtr3[n] = qaerwatsub3[n][jsub] * fcvt_wtr(); - qwtr4[n] = qwtr3[n]; - } - Real qaercw2[num_aerosol_ids][num_modes] = {}; - Real qaercw3[num_aerosol_ids][num_modes] = {}; - Real qnumcw3[num_modes] = {}; - Real qaercw4[num_aerosol_ids][num_modes] = {}; - Real qnumcw4[num_modes] = {}; - if (iscldy_subarea[jsub]) { - // only do cloud-borne for cloudy - for (int n = 0; n < num_modes; ++n) { - qnumcw3[n] = qqcwsub3[n][jsub] * fcvt_num(); - qnumcw4[n] = qnumcw3[n]; - for (int iaer = 0; iaer < num_aerosol_ids; ++iaer) { - qaercw2[iaer][n] = qqcwsub2[n][jsub] * fcvt_aer(iaer); - qaercw3[iaer][n] = qqcwsub3[n][jsub] * fcvt_aer(iaer); - qaercw4[iaer][n] = qaercw3[iaer][n]; - } - } - } - - Real qgas_delaa[num_gas_ids][nqtendaa()] = {}; - Real qnum_delaa[num_modes][nqtendaa()] = {}; - Real qnumcw_delaa[num_modes][nqqcwtendaa()] = {}; - Real qaer_delaa[num_aerosol_ids][num_modes][nqtendaa()] = {}; - Real qaercw_delaa[num_aerosol_ids][num_modes][nqqcwtendaa()] = {}; - - if (iscldy_subarea[jsub]) { - mam_amicphys_1subarea_cloudy(sub_config, nstep, deltat, - jsub, nsubarea, iscldy_subarea[jsub], afracsub[jsub], temp, pmid, - pdel, zmid, pblh, relhumsub[jsub], dgn_a, dgn_awet, wetdens, qgas1, - qgas3, qgas4, qgas_delaa, qnum3, qnum4, qnum_delaa, qaer2, qaer3, - qaer4, qaer_delaa, qwtr3, qwtr4, qnumcw3, qnumcw4, qnumcw_delaa, - qaercw2, qaercw3, qaercw4, qaercw_delaa); - } else { - mam_amicphys_1subarea_clear(sub_config, nstep, deltat, - jsub, nsubarea, iscldy_subarea[jsub], afracsub[jsub], temp, pmid, - pdel, zmid, pblh, relhumsub[jsub], dgn_a, dgn_awet, wetdens, qgas1, - qgas3, qgas4, qgas_delaa, qnum3, qnum4, qnum_delaa, qaer3, qaer4, - qaer_delaa, qwtr3, qwtr4); - // map gas/aer/num arrays (mix-ratio and del=change) back to sub-area - // arrays - - if (do_map_gas_sub) { - for (int igas = 0; igas < 2; ++igas) { - const int l = lmap_gas(igas); - qsub4[l][jsub] = qgas4[igas] / fcvt_gas(igas); - for (int i = 0; i < nqtendaa(); ++i) - qsub_tendaa[l][i][jsub] = - qgas_delaa[igas][i] / (fcvt_gas(igas) * deltat); - } - } - for (int n = 0; n < num_modes; ++n) { - qsub4[n][jsub] = qnum4[n] / fcvt_num(); - for (int i = 0; i < nqtendaa(); ++i) - qsub_tendaa[n][i][jsub] = qnum_delaa[n][i] / (fcvt_num() * deltat); - for (int iaer = 0; iaer < num_aerosol_ids; ++iaer) { - qsub4[iaer][jsub] = qaer4[iaer][n] / fcvt_aer(iaer); - for (int i = 0; i < nqtendaa(); ++i) - qsub_tendaa[iaer][i][jsub] = - qaer_delaa[iaer][n][i] / (fcvt_aer(iaer) * deltat); - } - qaerwatsub4[n][jsub] = qwtr4[n] / fcvt_wtr(); - - if (iscldy_subarea[jsub]) { - qqcwsub4[n][jsub] = qnumcw4[n] / fcvt_num(); - for (int i = 0; i < nqqcwtendaa(); ++i) - qqcwsub_tendaa[n][i][jsub] = - qnumcw_delaa[n][i] / (fcvt_num() * deltat); - for (int iaer = 0; iaer < num_aerosol_ids; ++iaer) { - qqcwsub4[iaer][jsub] = qaercw4[iaer][n] / fcvt_aer(iaer); - for (int i = 0; i < nqqcwtendaa(); ++i) - qqcwsub_tendaa[iaer][i][jsub] = - qaercw_delaa[iaer][n][i] / (fcvt_aer(iaer) * deltat); - } - } - } - } - } -} - -} // anonymous namespace - -KOKKOS_INLINE_FUNCTION -void modal_aero_amicphys_intr( - const AmicPhysConfig& config, const int nstep, const Real deltat, const Real t, const Real pmid, const Real pdel, - const Real zm, const Real pblh, const Real qv, const Real cld, - Real q[gas_pcnst()], Real qqcw[gas_pcnst()], const Real q_pregaschem[gas_pcnst()], - const Real q_precldchem[gas_pcnst()], const Real qqcw_precldchem[gas_pcnst()], - Real q_tendbb[gas_pcnst()][nqtendbb()], Real qqcw_tendbb[gas_pcnst()][nqtendbb()], - Real dgncur_a[AeroConfig::num_modes()], - Real dgncur_awet[AeroConfig::num_modes()], - Real wetdens_host[AeroConfig::num_modes()], - Real qaerwat[AeroConfig::num_modes()]) { - - /* - nstep ! model time-step number - nqtendbb ! dimension for q_tendbb - nqqcwtendbb ! dimension f - deltat ! - q(ncol,pver,pcnstxx) ! current tracer mixing ratios (TMRs) - these values are updated (so out /= in) - *** MUST BE #/kmol-air for number - *** MUST BE mol/mol-air for mass - *** NOTE ncol dimension - qqcw(ncol,pver,pcnstxx) - like q but for cloud-borner tracers - these values are updated - q_pregaschem(ncol,pver,pcnstxx) ! q TMRs before gas-phase - chemistry q_precldchem(ncol,pver,pcnstxx) ! q TMRs before cloud - chemistry qqcw_precldchem(ncol,pver,pcnstxx) ! qqcw TMRs before cloud - chemistry q_tendbb(ncol,pver,pcnstxx,nqtendbb()) ! TMR tendencies for - box-model diagnostic output qqcw_tendbb(ncol,pver,pcnstx t(pcols,pver) ! - temperature at model levels (K) pmid(pcols,pver) ! pressure at model - level centers (Pa) pdel(pcols,pver) ! pressure thickness of levels - (Pa) zm(pcols,pver) ! altitude (above ground) at level centers (m) - pblh(pcols) ! planetary boundary layer depth (m) - qv(pcols,pver) ! specific humidity (kg/kg) - cld(ncol,pver) ! cloud fraction (-) *** NOTE ncol dimension - dgncur_a(pcols,pver,ntot_amode) - dgncur_awet(pcols,pver,ntot_amode) - ! dry & wet geo. mean dia. (m) of - number distrib. wetdens_host(pcols,pver,ntot_amode) ! interstitial - aerosol wet density (kg/m3) - - qaerwat(pcols,pver,ntot_amode aerosol water mixing ratio (kg/kg, - NOT mol/mol) - - */ - - // !DESCRIPTION: - // calculates changes to gas and aerosol TMRs (tracer mixing ratios) from - // gas-aerosol exchange (condensation/evaporation) - // growth from smaller to larger modes (renaming) due to both - // condensation and cloud chemistry - // new particle nucleation - // coagulation - // transfer of particles from hydrophobic modes to hydrophilic modes - // (aging) - // due to condensation and coagulation - // - // the incoming mixing ratios (q and qqcw) are updated before output - // - // !REVISION HISTORY: - // RCE 07.04.13: Adapted from earlier version of CAM5 modal aerosol - // routines - // for these processes - // - - static constexpr int num_modes = AeroConfig::num_modes(); - - // qgcmN and qqcwgcmN (N=1:4) are grid-cell mean tracer mixing ratios - // (TMRs, mol/mol or #/kmol) - // N=1 - before gas-phase chemistry - // N=2 - before cloud chemistry - // N=3 - incoming values (before gas-aerosol exchange, newnuc, coag) - // N=4 - outgoing values (after gas-aerosol exchange, newnuc, coag) - - // qsubN and qqcwsubN (N=1:4) are TMRs in sub-areas - // currently there are just clear and cloudy sub-areas - // the N=1:4 have same meanings as for qgcmN - - // q_coltendaa and qqcw_coltendaa are column-integrated tendencies - // for different processes, which are output to history - // the processes are condensation/evaporation (and associated aging), - // renaming, coagulation, and nucleation - - for (int i = 0; i < gas_pcnst(); ++i) - for (int j = 0; j < nqtendbb(); ++j) - q_tendbb[i][j] = 0.0, qqcw_tendbb[i][j] = 0.0; - - // get saturation mixing ratio - // call qsat( t(1:ncol,1:pver), pmid(1:ncol,1:pvnner), & - // ev_sat(1:ncol,1:pver), qv_sat(1:ncol,1:pver) ) - const Real epsqs = haero::Constants::weight_ratio_h2o_air; - // Saturation vapor pressure - const Real ev_sat = conversions::vapor_saturation_pressure_magnus(t, pmid); - // Saturation specific humidity - const Real qv_sat = epsqs * ev_sat / (pmid - (1 - epsqs) * ev_sat); - - const Real relhumgcm = haero::max(0.0, haero::min(1.0, qv / qv_sat)); - - // Set up cloudy/clear subareas inside a grid cell - int nsubarea, ncldy_subarea, jclea, jcldy; - bool iscldy_subarea[maxsubarea()]; - Real afracsub[maxsubarea()]; - Real relhumsub[maxsubarea()]; - Real qsub1[gas_pcnst()][maxsubarea()]; - Real qsub2[gas_pcnst()][maxsubarea()]; - Real qsub3[gas_pcnst()][maxsubarea()]; - Real qqcwsub1[gas_pcnst()][maxsubarea()]; - Real qqcwsub2[gas_pcnst()][maxsubarea()]; - Real qqcwsub3[gas_pcnst()][maxsubarea()]; - // aerosol water mixing ratios (mol/mol) - Real qaerwatsub3[AeroConfig::num_modes()][maxsubarea()]; - construct_subareas_1gridcell(cld, relhumgcm, // in - q_pregaschem, q_precldchem, // in - qqcw_precldchem, // in - q, qqcw, // in - nsubarea, ncldy_subarea, jclea, jcldy, // out - iscldy_subarea, afracsub, relhumsub, // out - qsub1, qsub2, qsub3, // out - qqcwsub1, qqcwsub2, qqcwsub3, qaerwatsub3, // out - qaerwat // in - ); - - // Initialize the "after-amicphys" values - Real qsub4[gas_pcnst()][maxsubarea()] = {}; - Real qqcwsub4[gas_pcnst()][maxsubarea()] = {}; - Real qaerwatsub4[AeroConfig::num_modes()][maxsubarea()] = {}; - - // - // start integration - // - Real dgn_a[num_modes], dgn_awet[num_modes], wetdens[num_modes]; - for (int n = 0; n < num_modes; ++n) { - dgn_a[n] = dgncur_a[n]; - dgn_awet[n] = dgncur_awet[n]; - wetdens[n] = haero::max(1000.0, wetdens_host[n]); - } - Real qsub_tendaa[gas_pcnst()][nqtendaa()][maxsubarea()] = {}; - Real qqcwsub_tendaa[gas_pcnst()][nqqcwtendaa()][maxsubarea()] = {}; - mam_amicphys_1gridcell(config, nstep, deltat, - nsubarea, ncldy_subarea, iscldy_subarea, afracsub, t, - pmid, pdel, zm, pblh, relhumsub, dgn_a, dgn_awet, - wetdens, qsub1, qsub2, qqcwsub2, qsub3, qqcwsub3, - qaerwatsub3, qsub4, qqcwsub4, qaerwatsub4, qsub_tendaa, - qqcwsub_tendaa); - - // - // form new grid-mean mix-ratios - Real qgcm4[gas_pcnst()]; - Real qgcm_tendaa[gas_pcnst()][nqtendaa()]; - Real qaerwatgcm4[num_modes]; - if (nsubarea == 1) { - for (int i = 0; i < gas_pcnst(); ++i) - qgcm4[i] = qsub4[i][0]; - for (int i = 0; i < gas_pcnst(); ++i) - for (int j = 0; j < nqtendaa(); ++j) - qgcm_tendaa[i][j] = qsub_tendaa[i][j][0]; - for (int i = 0; i < num_modes; ++i) - qaerwatgcm4[i] = qaerwatsub4[i][0]; - } else { - for (int i = 0; i < gas_pcnst(); ++i) - qgcm4[i] = 0.0; - for (int i = 0; i < gas_pcnst(); ++i) - for (int j = 0; j < nqtendaa(); ++j) - qgcm_tendaa[i][j] = 0.0; - for (int n = 0; n < nsubarea; ++n) { - for (int i = 0; i < gas_pcnst(); ++i) - qgcm4[i] += qsub4[i][n] * afracsub[n]; - for (int i = 0; i < gas_pcnst(); ++i) - for (int j = 0; j < nqtendaa(); ++j) - qgcm_tendaa[i][j] = - qgcm_tendaa[i][j] + qsub_tendaa[i][j][n] * afracsub[n]; - } - for (int i = 0; i < num_modes; ++i) - // for aerosol water use the clear sub-area value - qaerwatgcm4[i] = qaerwatsub4[i][jclea - 1]; - } - Real qqcwgcm4[gas_pcnst()]; - Real qqcwgcm_tendaa[gas_pcnst()][nqqcwtendaa()]; - if (ncldy_subarea <= 0) { - for (int i = 0; i < gas_pcnst(); ++i) - qqcwgcm4[i] = haero::max(0.0, qqcw[i]); - for (int i = 0; i < gas_pcnst(); ++i) - for (int j = 0; j < nqqcwtendaa(); ++j) - qqcwgcm_tendaa[i][j] = 0.0; - } else if (nsubarea == 1) { - for (int i = 0; i < gas_pcnst(); ++i) - qqcwgcm4[i] = qqcwsub4[i][0]; - for (int i = 0; i < gas_pcnst(); ++i) - for (int j = 0; j < nqqcwtendaa(); ++j) - qqcwgcm_tendaa[i][j] = qqcwsub_tendaa[i][j][0]; - } else { - for (int i = 0; i < gas_pcnst(); ++i) - qqcwgcm4[i] = 0.0; - for (int i = 0; i < gas_pcnst(); ++i) - for (int j = 0; j < nqqcwtendaa(); ++j) - qqcwgcm_tendaa[i][j] = 0.0; - for (int n = 0; n < nsubarea; ++n) { - if (iscldy_subarea[n]) { - for (int i = 0; i < gas_pcnst(); ++i) - qqcwgcm4[i] += qqcwsub4[i][n] * afracsub[n]; - for (int i = 0; i < gas_pcnst(); ++i) - for (int j = 0; j < nqqcwtendaa(); ++j) - qqcwgcm_tendaa[i][j] += qqcwsub_tendaa[i][j][n] * afracsub[n]; - } - } - } - - for (int lmz = 0; lmz < gas_pcnst(); ++lmz) { - if (lmapcc_all(lmz) > 0) { - // HW, to ensure non-negative - q[lmz] = haero::max(qgcm4[lmz], 0.0); - if (lmapcc_all(lmz) >= lmapcc_val_aer()) { - // HW, to ensure non-negative - qqcw[lmz] = haero::max(qqcwgcm4[lmz], 0.0); - } - } - } - for (int i = 0; i < gas_pcnst(); ++i) { - if (iqtend_cond() < nqtendbb()) - q_tendbb[i][iqtend_cond()] = qgcm_tendaa[i][iqtend_cond()]; - if (iqtend_rnam() < nqtendbb()) - q_tendbb[i][iqtend_rnam()] = qgcm_tendaa[i][iqtend_rnam()]; - if (iqtend_nnuc() < nqtendbb()) - q_tendbb[i][iqtend_nnuc()] = qgcm_tendaa[i][iqtend_nnuc()]; - if (iqtend_coag() < nqtendbb()) - q_tendbb[i][iqtend_coag()] = qgcm_tendaa[i][iqtend_coag()]; - if (iqqcwtend_rnam() < nqqcwtendbb()) - qqcw_tendbb[i][iqqcwtend_rnam()] = qqcwgcm_tendaa[i][iqqcwtend_rnam()]; - } - for (int i = 0; i < num_modes; ++i) - qaerwat[i] = qaerwatgcm4[i]; -} - -} // namespace scream::impl diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index a3a5f746aa30..68f67f62c5c6 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -647,7 +647,7 @@ void compute_vertical_layer_heights(const Team& team, const auto p_mid = ekat::subview(dry_atm.p_mid, column_index); const auto T_mid = ekat::subview(dry_atm.T_mid, column_index); const auto pseudo_density = ekat::subview(dry_atm.p_del, column_index); - + // NOTE: we are using dry qv. Does calculate_dz require dry or wet? PF::calculate_dz(team, pseudo_density, p_mid, T_mid, qv, // inputs dz);//output @@ -811,196 +811,6 @@ void copy_view_lev_slice(haero::ThreadTeamPolicy team_policy, //inputs }); } -// Because CUDA C++ doesn't allow us to declare and use constants outside of -// KOKKOS_INLINE_FUNCTIONS, we define this macro that allows us to (re)define -// these constants where needed within two such functions so we don't define -// them inconsistently. Yes, it's the 21st century and we're still struggling -// with these basic things. -#define DECLARE_PROG_TRANSFER_CONSTANTS \ - /* mapping of constituent indices to aerosol modes */ \ - const auto Accum = mam4::ModeIndex::Accumulation; \ - const auto Aitken = mam4::ModeIndex::Aitken; \ - const auto Coarse = mam4::ModeIndex::Coarse; \ - const auto PC = mam4::ModeIndex::PrimaryCarbon; \ - const auto NoMode = mam4::ModeIndex::None; \ - static const mam4::ModeIndex mode_for_cnst[gas_pcnst()] = { \ - NoMode, NoMode, NoMode, NoMode, NoMode, NoMode, /* gases (not aerosols) */ \ - Accum, Accum, Accum, Accum, Accum, Accum, Accum, Accum, /* 7 aero species + NMR */ \ - Aitken, Aitken, Aitken, Aitken, Aitken, /* 4 aero species + NMR */ \ - Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, /* 7 aero species + NMR */ \ - PC, PC, PC, PC, /* 3 aero species + NMR */ \ - }; \ - /* mapping of constituent indices to aerosol species */ \ - const auto SOA = mam4::AeroId::SOA; \ - const auto SO4 = mam4::AeroId::SO4; \ - const auto POM = mam4::AeroId::POM; \ - const auto BC = mam4::AeroId::BC; \ - const auto NaCl = mam4::AeroId::NaCl; \ - const auto DST = mam4::AeroId::DST; \ - const auto MOM = mam4::AeroId::MOM; \ - const auto NoAero = mam4::AeroId::None; \ - static const mam4::AeroId aero_for_cnst[gas_pcnst()] = { \ - NoAero, NoAero, NoAero, NoAero, NoAero, NoAero, /* gases (not aerosols) */ \ - SO4, POM, SOA, BC, DST, NaCl, MOM, NoAero, /* accumulation mode */ \ - SO4, SOA, NaCl, MOM, NoAero, /* aitken mode */ \ - DST, NaCl, SO4, BC, POM, SOA, MOM, NoAero, /* coarse mode */ \ - POM, BC, MOM, NoAero, /* primary carbon mode */ \ - }; \ - /* mapping of constituent indices to gases */ \ - const auto O3 = mam4::GasId::O3; \ - const auto H2O2 = mam4::GasId::H2O2; \ - const auto H2SO4 = mam4::GasId::H2SO4; \ - const auto SO2 = mam4::GasId::SO2; \ - const auto DMS = mam4::GasId::DMS; \ - const auto SOAG = mam4::GasId::SOAG; \ - const auto NoGas = mam4::GasId::None; \ - static const mam4::GasId gas_for_cnst[gas_pcnst()] = { \ - O3, H2O2, H2SO4, SO2, DMS, SOAG, \ - NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, \ - NoGas, NoGas, NoGas, NoGas, NoGas, \ - NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, \ - NoGas, NoGas, NoGas, NoGas, \ - }; - -// Given a Prognostics object, transfers data for interstitial aerosols to the -// chemistry work array q, and cloudborne aerosols to the chemistry work array -// qqcw, both at vertical level k. The input and output quantities are stored as -// number/mass mixing ratios. -// NOTE: this mapping is chemistry-mechanism-specific (see mo_sim_dat.F90 -// NOTE: in the relevant preprocessed chemical mechanism) -// NOTE: see mam4xx/aero_modes.hpp to interpret these mode/aerosol/gas -// NOTE: indices -KOKKOS_INLINE_FUNCTION -void transfer_prognostics_to_work_arrays(const mam4::Prognostics &progs, - const int k, - Real q[gas_pcnst()], - Real qqcw[gas_pcnst()]) { - DECLARE_PROG_TRANSFER_CONSTANTS - - // copy number/mass mixing ratios from progs to q and qqcw at level k, - // converting them to VMR - for (int i = 0; i < gas_pcnst(); ++i) { - auto mode_index = mode_for_cnst[i]; - auto aero_id = aero_for_cnst[i]; - auto gas_id = gas_for_cnst[i]; - if (gas_id != NoGas) { // constituent is a gas - int g = static_cast(gas_id); - q[i] = progs.q_gas[g](k); - qqcw[i] = progs.q_gas[g](k); - } else { - int m = static_cast(mode_index); - if (aero_id != NoAero) { // constituent is an aerosol species - int a = aerosol_index_for_mode(mode_index, aero_id); - q[i] = progs.q_aero_i[m][a](k); - qqcw[i] = progs.q_aero_c[m][a](k); - } else { // constituent is a modal number mixing ratio - int m = static_cast(mode_index); - q[i] = progs.n_mode_i[m](k); - qqcw[i] = progs.n_mode_c[m](k); - } - } - } -} - -// converts the quantities in the work arrays q and qqcw from mass/number -// mixing ratios to volume/number mixing ratios -KOKKOS_INLINE_FUNCTION -void convert_work_arrays_to_vmr(const Real q[gas_pcnst()], - const Real qqcw[gas_pcnst()], - Real vmr[gas_pcnst()], - Real vmrcw[gas_pcnst()]) { - DECLARE_PROG_TRANSFER_CONSTANTS - - for (int i = 0; i < gas_pcnst(); ++i) { - auto mode_index = mode_for_cnst[i]; - auto aero_id = aero_for_cnst[i]; - auto gas_id = gas_for_cnst[i]; - if (gas_id != NoGas) { // constituent is a gas - int g = static_cast(gas_id); - const Real mw = mam4::gas_species(g).molecular_weight; - vmr[i] = mam4::conversions::vmr_from_mmr(q[i], mw); - vmrcw[i] = mam4::conversions::vmr_from_mmr(qqcw[i], mw); - } else { - if (aero_id != NoAero) { // constituent is an aerosol species - int a = aerosol_index_for_mode(mode_index, aero_id); - const Real mw = mam4::aero_species(a).molecular_weight; - vmr[i] = mam4::conversions::vmr_from_mmr(q[i], mw); - vmrcw[i] = mam4::conversions::vmr_from_mmr(qqcw[i], mw); - } else { // constituent is a modal number mixing ratio - vmr[i] = q[i]; - vmrcw[i] = qqcw[i]; - } - } - } -} - -// converts the quantities in the work arrays vmrs and vmrscw from mass/number -// mixing ratios to volume/number mixing ratios -KOKKOS_INLINE_FUNCTION -void convert_work_arrays_to_mmr(const Real vmr[gas_pcnst()], - const Real vmrcw[gas_pcnst()], - Real q[gas_pcnst()], - Real qqcw[gas_pcnst()]) { - DECLARE_PROG_TRANSFER_CONSTANTS - - for (int i = 0; i < gas_pcnst(); ++i) { - auto mode_index = mode_for_cnst[i]; - auto aero_id = aero_for_cnst[i]; - auto gas_id = gas_for_cnst[i]; - if (gas_id != NoGas) { // constituent is a gas - int g = static_cast(gas_id); - const Real mw = mam4::gas_species(g).molecular_weight; - q[i] = mam4::conversions::mmr_from_vmr(vmr[i], mw); - qqcw[i] = mam4::conversions::mmr_from_vmr(vmrcw[i], mw); - } else { - if (aero_id != NoAero) { // constituent is an aerosol species - int a = aerosol_index_for_mode(mode_index, aero_id); - const Real mw = mam4::aero_species(a).molecular_weight; - q[i] = mam4::conversions::mmr_from_vmr(vmr[i], mw); - qqcw[i] = mam4::conversions::mmr_from_vmr(vmrcw[i], mw); - } else { // constituent is a modal number mixing ratio - q[i] = vmr[i]; - qqcw[i] = vmrcw[i]; - } - } - } -} - -// Given work arrays with interstitial and cloudborne aerosol data, transfers -// them to the given Prognostics object at the kth vertical level. This is the -// "inverse operator" for transfer_prognostics_to_work_arrays, above. -KOKKOS_INLINE_FUNCTION -void transfer_work_arrays_to_prognostics(const Real q[gas_pcnst()], - const Real qqcw[gas_pcnst()], - mam4::Prognostics &progs, const int k) { - DECLARE_PROG_TRANSFER_CONSTANTS - - // copy number/mass mixing ratios from progs to q and qqcw at level k, - // converting them to VMR - for (int i = 0; i < gas_pcnst(); ++i) { - auto mode_index = mode_for_cnst[i]; - auto aero_id = aero_for_cnst[i]; - auto gas_id = gas_for_cnst[i]; - if (gas_id != NoGas) { // constituent is a gas - int g = static_cast(gas_id); - progs.q_gas[g](k) = q[i]; - } else { - int m = static_cast(mode_index); - if (aero_id != NoAero) { // constituent is an aerosol species - int a = aerosol_index_for_mode(mode_index, aero_id); - progs.q_aero_i[m][a](k) = q[i]; - progs.q_aero_c[m][a](k) = qqcw[i]; - } else { // constituent is a modal number mixing ratio - int m = static_cast(mode_index); - progs.n_mode_i[m](k) = q[i]; - progs.n_mode_c[m](k) = qqcw[i]; - } - } - } -} - -#undef DECLARE_PROG_TRANSFER_CONSTANTS - } // namespace scream::mam_coupling #endif diff --git a/components/eamxx/src/physics/mam/readfiles/find_season_index_utils.hpp b/components/eamxx/src/physics/mam/readfiles/find_season_index_utils.hpp new file mode 100644 index 000000000000..2e930cc65cbf --- /dev/null +++ b/components/eamxx/src/physics/mam/readfiles/find_season_index_utils.hpp @@ -0,0 +1,75 @@ +#ifndef EAMXX_MAM_FIND_SEASON_INDEX_UTILS +#define EAMXX_MAM_FIND_SEASON_INDEX_UTILS + +#include +#include + +#include "share/io/scorpio_input.hpp" +#include "share/io/scream_scorpio_interface.hpp" + +namespace scream::mam_coupling { + +// views for single- and multi-column data + +using const_view_1d = typename KT::template view_1d; +using view_int_2d = typename KT::template view_2d; + +using view_1d_host = typename KT::view_1d::HostMirror; +using view_int_3d_host = typename KT::view_3d::HostMirror; +using view_int_2d_host = typename KT::view_2d::HostMirror; + +/** + * @brief Reads the season index from the given file and computes the season + * indices based on latitudes. + * + * @param[in] season_wes_file The path to the season_wes.nc file. + * @param[in] clat A 1D view of latitude values in degrees. + * @param[out] index_season_lai A 2D view to store the computed season indices. + * Note that indices are in C++ (starting from zero). + */ + +inline void find_season_index_reader(const std::string &season_wes_file, + const const_view_1d &clat, + view_int_2d &index_season_lai) { + const int plon = clat.extent(0); + scorpio::register_file(season_wes_file, scorpio::Read); + + const int nlat_lai = scorpio::get_dimlen(season_wes_file, "lat"); + const int npft_lai = scorpio::get_dimlen(season_wes_file, "pft"); + + view_1d_host lat_lai("lat_lai", nlat_lai); + view_int_2d_host wk_lai_temp("wk_lai", npft_lai, nlat_lai); + view_int_3d_host wk_lai("wk_lai", nlat_lai, npft_lai, 12); + + scorpio::read_var(season_wes_file, "lat", lat_lai.data()); + + Kokkos::MDRangePolicy> + policy_wk_lai({0, 0}, {nlat_lai, npft_lai}); + + // loop over time to get all 12 instantence of season_wes + for(int itime = 0; itime < 12; ++itime) { + scorpio::read_var(season_wes_file, "season_wes", wk_lai_temp.data(), itime); + // copy data from wk_lai_temp to wk_lai. + // NOTE: season_wes has different layout that wk_lai + Kokkos::parallel_for("copy_to_wk_lai", policy_wk_lai, + [&](const int j, const int k) { + wk_lai(j, k, itime) = wk_lai_temp(k, j); + }); + Kokkos::fence(); + } + scorpio::release_file(season_wes_file); + + index_season_lai = view_int_2d("index_season_lai", plon, 12); + const view_int_2d_host index_season_lai_host = + Kokkos::create_mirror_view(index_season_lai); + + const view_1d_host clat_host = Kokkos::create_mirror_view(clat); + Kokkos::deep_copy(clat_host, clat); + + // Computation is performed on the host + mam4::mo_drydep::find_season_index(clat_host, lat_lai, nlat_lai, wk_lai, + index_season_lai_host); + Kokkos::deep_copy(index_season_lai, index_season_lai_host); +} +} // namespace scream::mam_coupling +#endif // diff --git a/components/eamxx/src/physics/mam/readfiles/fractional_land_use.hpp b/components/eamxx/src/physics/mam/readfiles/fractional_land_use.hpp new file mode 100644 index 000000000000..1dba53c38ed7 --- /dev/null +++ b/components/eamxx/src/physics/mam/readfiles/fractional_land_use.hpp @@ -0,0 +1,59 @@ +#ifndef FRACTIONAL_LANDUSE_HPP +#define FRACTIONAL_LANDUSE_HPP + +// For AtmosphereInput +#include "share/io/scorpio_input.hpp" + +namespace scream { +namespace frac_landuse { + +template +struct fracLandUseFunctions { + using Device = DeviceType; + + using KT = KokkosTypes; + using const_view_2d = typename KT::template view_2d; + + // ------------------------------------------------------------------------------------------- + // ------------------------------------------------------------------------------------------- + + // Fractional land use routines + static std::shared_ptr create_horiz_remapper( + const std::shared_ptr &model_grid, + const std::string &fracLandUse_data_file, const std::string &map_file, + const std::string &field_name, const std::string &dim_name1, + const std::string &dim_name2); + + // ------------------------------------------------------------------------------------------- + // ------------------------------------------------------------------------------------------- + + static std::shared_ptr create_data_reader( + const std::shared_ptr &horiz_remapper, + const std::string &data_file); + + // ------------------------------------------------------------------------------------------- + // ------------------------------------------------------------------------------------------- + + static void update_frac_land_use_data_from_file( + std::shared_ptr &scorpio_reader, + AbstractRemapper &horiz_interp, const_view_2d &input); + + // ------------------------------------------------------------------------------------------- + // ------------------------------------------------------------------------------------------- + + static void init_frac_landuse_file_read( + const int ncol, const std::string field_name, const std::string dim_name1, + const std::string dim_name2, + const std::shared_ptr &grid, + const std::string &data_file, const std::string &mapping_file, + // output + std::shared_ptr &FracLandUseHorizInterp, + std::shared_ptr &FracLandUseDataReader); + +}; // struct fracLandUseFunctions + +} // namespace frac_landuse +} // namespace scream +#endif // FRACTIONAL_LANDUSE_HPP + +#include "fractional_land_use_impl.hpp" \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/readfiles/fractional_land_use_impl.hpp b/components/eamxx/src/physics/mam/readfiles/fractional_land_use_impl.hpp new file mode 100644 index 000000000000..8bddd390e190 --- /dev/null +++ b/components/eamxx/src/physics/mam/readfiles/fractional_land_use_impl.hpp @@ -0,0 +1,148 @@ +#ifndef FRACTIONAL_LANDUSE_IMPL_HPP +#define FRACTIONAL_LANDUSE_IMPL_HPP + +#include "share/grid/remap/identity_remapper.hpp" +#include "share/grid/remap/refining_remapper_p2p.hpp" +#include "share/io/scream_scorpio_interface.hpp" +#include "share/util/scream_timing.hpp" + +namespace scream { +namespace frac_landuse { + +template +std::shared_ptr +fracLandUseFunctions::create_horiz_remapper( + const std::shared_ptr &model_grid, + const std::string &data_file, const std::string &map_file, + const std::string &field_name, const std::string &dim_name1, + const std::string &dim_name2) { + using namespace ShortFieldTagsNames; + + scorpio::register_file(data_file, scorpio::Read); + const int ncols_data = scorpio::get_dimlen(data_file, dim_name1); + const int nclass_data = scorpio::get_dimlen(data_file, dim_name2); + + scorpio::release_file(data_file); + + // We could use model_grid directly if using same num levels, + // but since shallow clones are cheap, we may as well do it (less lines of + // code) + auto horiz_interp_tgt_grid = + model_grid->clone("frac_land_use_horiz_interp_tgt_grid", true); + + const int ncols_model = model_grid->get_num_global_dofs(); + std::shared_ptr remapper; + if(ncols_data == ncols_model) { + remapper = std::make_shared( + horiz_interp_tgt_grid, IdentityRemapper::SrcAliasTgt); + } else { + EKAT_REQUIRE_MSG(ncols_data <= ncols_model, + "Error! We do not allow to coarsen fractional land use " + "data to fit the model. We only allow\n" + " fractional land use data to be at the same or " + "coarser resolution as the model.\n"); + // We must have a valid map file + EKAT_REQUIRE_MSG(map_file != "", + "ERROR: fractional land use data is on a different grid " + "than the model one,\n" + " but remap file is missing from fractional " + "land use parameter list."); + + remapper = + std::make_shared(horiz_interp_tgt_grid, map_file); + } + + remapper->registration_begins(); + + const auto tgt_grid = remapper->get_tgt_grid(); + + const auto layout_2d = tgt_grid->get_2d_vector_layout(nclass_data, "class"); + const auto nondim = ekat::units::Units::nondimensional(); + + Field fractional_land_use( + FieldIdentifier(field_name, layout_2d, nondim, tgt_grid->name())); + fractional_land_use.allocate_view(); + + remapper->register_field_from_tgt(fractional_land_use); + + remapper->registration_ends(); + + return remapper; +} // create_horiz_remapper + +// ------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------- + +template +std::shared_ptr fracLandUseFunctions::create_data_reader( + const std::shared_ptr &horiz_remapper, + const std::string &data_file) { + std::vector io_fields; + for(int i = 0; i < horiz_remapper->get_num_fields(); ++i) { + io_fields.push_back(horiz_remapper->get_src_field(i)); + } + const auto io_grid = horiz_remapper->get_src_grid(); + return std::make_shared(data_file, io_grid, io_fields, true); +} // create_data_reader + +// ------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------- + +template +void fracLandUseFunctions::update_frac_land_use_data_from_file( + std::shared_ptr &scorpio_reader, + AbstractRemapper &horiz_interp, const_view_2d &input) { + start_timer("EAMxx::FracLandUse::update_frac_land_use_data_from_file"); + + // 1. Read from file + start_timer( + "EAMxx::FracLandUse::update_frac_land_use_data_from_file::read_data"); + scorpio_reader->read_variables(); + stop_timer( + "EAMxx::FracLandUse::update_frac_land_use_data_from_file::read_data"); + + // 2. Run the horiz remapper (it is a do-nothing op if FracLandUse data is on + // same grid as model) + start_timer( + "EAMxx::FracLandUse::update_frac_land_use_data_from_file::horiz_remap"); + horiz_interp.remap(/*forward = */ true); + stop_timer( + "EAMxx::FracLandUse::update_frac_land_use_data_from_file::horiz_remap"); + + // 3. Get the tgt field of the remapper + start_timer( + "EAMxx::FracLandUse::update_frac_land_use_data_from_file::get_field"); + // Recall, the fields are registered in the order: + // Read the field from the file + input = horiz_interp.get_tgt_field(0).get_view(); + stop_timer( + "EAMxx::FracLandUse::update_frac_land_use_data_from_file::get_field"); + + stop_timer("EAMxx::FracLandUse::update_frac_land_use_data_from_file"); + +} // END update_frac_landuse_data_from_file + +// ------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------- + +template +void fracLandUseFunctions::init_frac_landuse_file_read( + const int ncol, const std::string field_name, const std::string dim_name1, + const std::string dim_name2, + const std::shared_ptr &grid, + const std::string &data_file, const std::string &mapping_file, + // output + std::shared_ptr &FracLandUseHorizInterp, + std::shared_ptr &FracLandUseDataReader) { + // Init horizontal remap + FracLandUseHorizInterp = create_horiz_remapper( + grid, data_file, mapping_file, field_name, dim_name1, dim_name2); + + // Create reader (an AtmosphereInput object) + FracLandUseDataReader = create_data_reader(FracLandUseHorizInterp, data_file); +} // init_frac_landuse_file_read + +} // namespace frac_landuse +} // namespace scream + +#endif // FRACTIONAL_LANDUSE_IMPL_HPP diff --git a/components/eamxx/src/physics/mam/readfiles/marine_organics.hpp b/components/eamxx/src/physics/mam/readfiles/marine_organics.hpp new file mode 100644 index 000000000000..a04dff129f49 --- /dev/null +++ b/components/eamxx/src/physics/mam/readfiles/marine_organics.hpp @@ -0,0 +1,133 @@ +#ifndef MARINE_ORGANICS_HPP +#define MARINE_ORGANICS_HPP + +// For AtmosphereInput +#include "share/io/scorpio_input.hpp" + +namespace scream { +namespace marine_organics { + +template +struct marineOrganicsFunctions { + using Device = DeviceType; + + using KT = KokkosTypes; + using MemberType = typename KT::MemberType; + using view_2d = typename KT::template view_2d; + + // ------------------------------------------------------------------------------------------- + struct marineOrganicsTimeState { + marineOrganicsTimeState() = default; + // Whether the timestate has been initialized. + // The current month + int current_month = -1; + // Julian Date for the beginning of the month, as defined in + // /src/share/util/scream_time_stamp.hpp + // See this file for definition of Julian Date. + Real t_beg_month; + // Current simulation Julian Date + Real t_now; + // Number of days in the current month, cast as a Real + Real days_this_month; + }; // marineOrganicsTimeState + + struct marineOrganicsData { + marineOrganicsData() = default; + marineOrganicsData(const int &ncol_, const int &nfields_) + : ncols(ncol_), nsectors(nfields_) { + init(ncols, nsectors, true); + } + + void init(const int &ncol, const int &nsector, const bool allocate) { + ncols = ncol; + nsectors = nsector; + if(allocate) emiss_sectors = view_2d("morgAllSectors", nsectors, ncols); + } // marineOrganicsData init + + // Basic spatial dimensions of the data + int ncols, nsectors; + view_2d emiss_sectors; + }; // marineOrganicsData + + // ------------------------------------------------------------------------------------------- + struct marineOrganicsInput { + marineOrganicsInput() = default; + marineOrganicsInput(const int &ncols_, const int &nfields_) { + init(ncols_, nfields_); + } + + void init(const int &ncols_, const int &nfields_) { + data.init(ncols_, nfields_, true); + } + marineOrganicsData data; // All marineOrganics fields + }; // marineOrganicsInput + + // The output is really just marineOrganicsData, but for clarity it might + // help to see a marineOrganicsOutput along a marineOrganicsInput in functions + // signatures + using marineOrganicsOutput = marineOrganicsData; + + // ------------------------------------------------------------------------------------------- + static std::shared_ptr create_horiz_remapper( + const std::shared_ptr &model_grid, + const std::string &marineOrganics_data_file, const std::string &map_file, + const std::vector &field_name, const std::string &dim_name1); + + // ------------------------------------------------------------------------------------------- + static std::shared_ptr create_data_reader( + const std::shared_ptr &horiz_remapper, + const std::string &data_file); + + // ------------------------------------------------------------------------------------------- + static void update_marine_organics_data_from_file( + std::shared_ptr &scorpio_reader, + const util::TimeStamp &ts, + const int &time_index, // zero-based + AbstractRemapper &horiz_interp, + marineOrganicsInput &marineOrganics_input); + + // ------------------------------------------------------------------------------------------- + static void update_marine_organics_timestate( + std::shared_ptr &scorpio_reader, + const util::TimeStamp &ts, AbstractRemapper &horiz_interp, + marineOrganicsTimeState &time_state, marineOrganicsInput &beg, + marineOrganicsInput &end); + + // ------------------------------------------------------------------------------------------- + static void marineOrganics_main(const marineOrganicsTimeState &time_state, + const marineOrganicsInput &data_beg, + const marineOrganicsInput &data_end, + const marineOrganicsOutput &data_out); + + // ------------------------------------------------------------------------------------------- + static void perform_time_interpolation( + const marineOrganicsTimeState &time_state, + const marineOrganicsInput &data_beg, const marineOrganicsInput &data_end, + const marineOrganicsOutput &data_out); + + // ------------------------------------------------------------------------------------------- + // Performs convex interpolation of x0 and x1 at point t + template + KOKKOS_INLINE_FUNCTION static ScalarX linear_interp(const ScalarX &x0, + const ScalarX &x1, + const ScalarT &t); + + // ------------------------------------------------------------------------------------------- + static void init_marine_organics_file_read( + const int &ncol, const std::vector &field_name, + const std::string &dim_name1, + const std::shared_ptr &grid, + const std::string &data_file, const std::string &mapping_file, + // output + std::shared_ptr &marineOrganicsHorizInterp, + marineOrganicsInput &morg_data_start_, + marineOrganicsInput &morg_data_end_, marineOrganicsData &morg_data_out_, + std::shared_ptr &marineOrganicsDataReader); + +}; // struct marineOrganicsFunctions + +} // namespace marine_organics +} // namespace scream +#endif // MARINE_ORGANICS_HPP + +#include "marine_organics_impl.hpp" \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/readfiles/marine_organics_impl.hpp b/components/eamxx/src/physics/mam/readfiles/marine_organics_impl.hpp new file mode 100644 index 000000000000..6758dccdef5a --- /dev/null +++ b/components/eamxx/src/physics/mam/readfiles/marine_organics_impl.hpp @@ -0,0 +1,294 @@ +#ifndef MARINE_ORGANICS_IMPL_HPP +#define MARINE_ORGANICS_IMPL_HPP + +#include "share/grid/remap/identity_remapper.hpp" +#include "share/grid/remap/refining_remapper_p2p.hpp" +#include "share/io/scream_scorpio_interface.hpp" +#include "share/util/scream_timing.hpp" + +namespace scream { +namespace marine_organics { + +template +std::shared_ptr +marineOrganicsFunctions::create_horiz_remapper( + const std::shared_ptr &model_grid, + const std::string &data_file, const std::string &map_file, + const std::vector &field_name, const std::string &dim_name1) { + using namespace ShortFieldTagsNames; + + scorpio::register_file(data_file, scorpio::Read); + const int ncols_data = scorpio::get_dimlen(data_file, dim_name1); + + scorpio::release_file(data_file); + + // Since shallow clones are cheap, we may as well do it (less lines of + // code) + auto horiz_interp_tgt_grid = + model_grid->clone("marine_organics_horiz_interp_tgt_grid", true); + + const int ncols_model = model_grid->get_num_global_dofs(); + std::shared_ptr remapper; + if(ncols_data == ncols_model) { + remapper = std::make_shared( + horiz_interp_tgt_grid, IdentityRemapper::SrcAliasTgt); + } else { + EKAT_REQUIRE_MSG(ncols_data <= ncols_model, + "Error! We do not allow to coarsen marine organics " + "data to fit the model. We only allow\n" + " marine organics data to be at the same or " + "coarser resolution as the model.\n"); + // We must have a valid map file + EKAT_REQUIRE_MSG(map_file != "", + "ERROR: marine organics data is on a different grid " + "than the model one,\n" + " but remap file is missing from marine organics " + "parameter list."); + + remapper = + std::make_shared(horiz_interp_tgt_grid, map_file); + } + + remapper->registration_begins(); + + const auto tgt_grid = remapper->get_tgt_grid(); + + const auto layout_2d = tgt_grid->get_2d_scalar_layout(); + using namespace ekat::units; + using namespace ekat::prefixes; + Units umolC(micro * mol, "umol C"); + + std::vector fields_vector; + + const int field_size = field_name.size(); + for(int icomp = 0; icomp < field_size; ++icomp) { + auto comp_name = field_name[icomp]; + // set and allocate fields + Field f(FieldIdentifier(comp_name, layout_2d, umolC, tgt_grid->name())); + f.allocate_view(); + fields_vector.push_back(f); + remapper->register_field_from_tgt(f); + } + + remapper->registration_ends(); + + return remapper; + +} // create_horiz_remapper + +// ------------------------------------------------------------------------------------------- +template +std::shared_ptr +marineOrganicsFunctions::create_data_reader( + const std::shared_ptr &horiz_remapper, + const std::string &data_file) { + std::vector io_fields; + for(int ifld = 0; ifld < horiz_remapper->get_num_fields(); ++ifld) { + io_fields.push_back(horiz_remapper->get_src_field(ifld)); + } + const auto io_grid = horiz_remapper->get_src_grid(); + return std::make_shared(data_file, io_grid, io_fields, true); +} // create_data_reader + +// ------------------------------------------------------------------------------------------- +template +void marineOrganicsFunctions::update_marine_organics_data_from_file( + std::shared_ptr &scorpio_reader, const util::TimeStamp &ts, + const int &time_index, // zero-based + AbstractRemapper &horiz_interp, marineOrganicsInput &marineOrganics_input) { + start_timer("EAMxx::marineOrganics::update_marine_organics_data_from_file"); + + // 1. Read from file + start_timer( + "EAMxx::marineOrganics::update_marine_organics_data_from_file::read_" + "data"); + scorpio_reader->read_variables(); + stop_timer( + "EAMxx::marineOrganics::update_marine_organics_data_from_file::read_" + "data"); + + // 2. Run the horiz remapper (it is a do-nothing op if marineOrganics data is + // on same grid as model) + start_timer( + "EAMxx::marineOrganics::update_marine_organics_data_from_file::horiz_" + "remap"); + horiz_interp.remap(/*forward = */ true); + stop_timer( + "EAMxx::marineOrganics::update_marine_organics_data_from_file::horiz_" + "remap"); + + // 3. Get the tgt field of the remapper + start_timer( + "EAMxx::marineOrganics::update_marine_organics_data_from_file::get_" + "field"); + // Recall, the fields are registered in the order: + // Read the field from the file + + for(int ifld = 0; ifld < horiz_interp.get_num_fields(); ++ifld) { + auto sector = horiz_interp.get_tgt_field(ifld).get_view(); + const auto emiss = Kokkos::subview(marineOrganics_input.data.emiss_sectors, + ifld, Kokkos::ALL()); + Kokkos::deep_copy(emiss, sector); + } + + Kokkos::fence(); + + stop_timer( + "EAMxx::marineOrganics::update_marine_organics_data_from_file::get_" + "field"); + + stop_timer("EAMxx::marineOrganics::update_marine_organics_data_from_file"); + +} // END update_marine_organics_data_from_file + +// ------------------------------------------------------------------------------------------- +template +void marineOrganicsFunctions::update_marine_organics_timestate( + std::shared_ptr &scorpio_reader, const util::TimeStamp &ts, + AbstractRemapper &horiz_interp, marineOrganicsTimeState &time_state, + marineOrganicsInput &beg, marineOrganicsInput &end) { + // Now we check if we have to update the data that changes monthly + // NOTE: This means that marineOrganics assumes monthly data to update. Not + // any other frequency. + const auto month = ts.get_month() - 1; // Make it 0-based + if(month != time_state.current_month) { + // Update the marineOrganics time state information + time_state.current_month = month; + time_state.t_beg_month = ts.curr_month_beg().frac_of_year_in_days(); + time_state.days_this_month = ts.days_in_curr_month(); + + // Copy end'data into beg'data, and read in the new + // end + std::swap(beg, end); + + // Update the marineOrganics forcing data for this month and next month + // Start by copying next months data to this months data structure. + // NOTE: If the timestep is bigger than monthly this could cause the wrong + // values + // to be assigned. A timestep greater than a month is very unlikely + // so we will proceed. + int next_month = (time_state.current_month + 1) % 12; + update_marine_organics_data_from_file(scorpio_reader, ts, next_month, + horiz_interp, end); + } + +} // END updata_marine_organics_timestate + +// ------------------------------------------------------------------------------------------- +template +template +KOKKOS_INLINE_FUNCTION ScalarX marineOrganicsFunctions::linear_interp( + const ScalarX &x0, const ScalarX &x1, const ScalarT &t) { + return (1 - t) * x0 + t * x1; +} // linear_interp + +// ------------------------------------------------------------------------------------------- +template +void marineOrganicsFunctions::perform_time_interpolation( + const marineOrganicsTimeState &time_state, + const marineOrganicsInput &data_beg, const marineOrganicsInput &data_end, + const marineOrganicsOutput &data_out) { + using ExeSpace = typename KT::ExeSpace; + using ESU = ekat::ExeSpaceUtils; + + // Gather time stamp info + auto &t_now = time_state.t_now; + auto &t_beg = time_state.t_beg_month; + auto &delta_t = time_state.days_this_month; + + // At this stage, begin/end must have the same dimensions + EKAT_REQUIRE(data_end.data.ncols == data_beg.data.ncols); + + auto delta_t_fraction = (t_now - t_beg) / delta_t; + + EKAT_REQUIRE_MSG(delta_t_fraction >= 0 && delta_t_fraction <= 1, + "Error! Convex interpolation with coefficient out of " + "[0,1].\n t_now : " + + std::to_string(t_now) + + "\n" + " t_beg : " + + std::to_string(t_beg) + + "\n delta_t: " + std::to_string(delta_t) + "\n"); + + const int nsectors = data_beg.data.nsectors; + const int ncols = data_beg.data.ncols; + using ExeSpace = typename KT::ExeSpace; + using ESU = ekat::ExeSpaceUtils; + const auto policy = ESU::get_default_team_policy(ncols, nsectors); + + Kokkos::parallel_for( + policy, KOKKOS_LAMBDA(const MemberType &team) { + const int icol = team.league_rank(); // column index + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, 0u, nsectors), [&](int isec) { + const auto beg = data_beg.data.emiss_sectors(isec, icol); + const auto end = data_end.data.emiss_sectors(isec, icol); + data_out.emiss_sectors(isec, icol) = + linear_interp(beg, end, delta_t_fraction); + }); + }); + Kokkos::fence(); + +} // perform_time_interpolation + +// ------------------------------------------------------------------------------------------- +template +void marineOrganicsFunctions::marineOrganics_main( + const marineOrganicsTimeState &time_state, + const marineOrganicsInput &data_beg, const marineOrganicsInput &data_end, + const marineOrganicsOutput &data_out) { + // Beg/End/Tmp month must have all sizes matching + + EKAT_REQUIRE_MSG( + data_end.data.ncols == data_beg.data.ncols, + "Error! marineOrganicsInput data structs must have the same number of " + "columns.\n"); + + // Horiz interpolation can be expensive, and does not depend on the particular + // time of the month, so it can be done ONCE per month, *outside* + // marineOrganics_main (when updating the beg/end states, reading them from + // file). + EKAT_REQUIRE_MSG(data_end.data.ncols == data_out.ncols, + "Error! Horizontal interpolation is performed *before* " + "calling marineOrganics_main,\n" + " marineOrganicsInput and marineOrganicsOutput data " + "structs must have the " + "same number columns " + << data_end.data.ncols << " " << data_out.ncols + << ".\n"); + + // Step 1. Perform time interpolation + perform_time_interpolation(time_state, data_beg, data_end, data_out); +} // marineOrganics_main + +// ------------------------------------------------------------------------------------------- +template +void marineOrganicsFunctions::init_marine_organics_file_read( + const int &ncol, const std::vector &field_name, + const std::string &dim_name1, + const std::shared_ptr &grid, + const std::string &data_file, const std::string &mapping_file, + // output + std::shared_ptr &marineOrganicsHorizInterp, + marineOrganicsInput &data_start_, marineOrganicsInput &data_end_, + marineOrganicsData &data_out_, + std::shared_ptr &marineOrganicsDataReader) { + // Init horizontal remap + + marineOrganicsHorizInterp = create_horiz_remapper( + grid, data_file, mapping_file, field_name, dim_name1); + + // Initialize the size of start/end/out data structures + data_start_ = marineOrganicsInput(ncol, field_name.size()); + data_end_ = marineOrganicsInput(ncol, field_name.size()); + data_out_.init(ncol, field_name.size(), true); + + // Create reader (an AtmosphereInput object) + marineOrganicsDataReader = + create_data_reader(marineOrganicsHorizInterp, data_file); + +} // init_marine_organics_file_read +} // namespace marine_organics +} // namespace scream + +#endif // MARINE_ORGANICS_IMPL_HPP diff --git a/components/eamxx/src/physics/mam/readfiles/photo_table_utils.cpp b/components/eamxx/src/physics/mam/readfiles/photo_table_utils.cpp new file mode 100644 index 000000000000..ab5a05092745 --- /dev/null +++ b/components/eamxx/src/physics/mam/readfiles/photo_table_utils.cpp @@ -0,0 +1,151 @@ +#include + +namespace scream::impl { + +// number of photolysis reactions +using mam4::mo_photo::phtcnt; + +using HostView1D = haero::DeviceType::view_1d::HostMirror; +using HostViewInt1D = haero::DeviceType::view_1d::HostMirror; + +//------------------------------------------------------------------------- +// Reading the photolysis table +//------------------------------------------------------------------------- + +// ON HOST (MPI root only), sets the lng_indexer and pht_alias_mult_1 host views +// according to parameters in our (hardwired) chemical mechanism + +std::vector populate_etfphot_from_e3sm_case() { + // We obtained these values from an e3sm simulations. + // We should only use this function on Host. + std::vector etfphot_data = { + 7.5691227E+11, 8.6525905E+11, 1.0355749E+12, 1.1846288E+12, 2.1524405E+12, + 3.2362584E+12, 3.7289849E+12, 4.4204330E+12, 4.6835350E+12, 6.1217728E+12, + 4.5575051E+12, 5.3491446E+12, 4.7016063E+12, 5.4281722E+12, 4.5023968E+12, + 6.8931981E+12, 6.2012647E+12, 6.1430771E+12, 5.7820385E+12, 7.6770646E+12, + 1.3966509E+13, 1.2105348E+13, 2.8588980E+13, 3.2160821E+13, 2.4978066E+13, + 2.7825401E+13, 2.3276451E+13, 3.6343684E+13, 6.1787886E+13, 7.8009914E+13, + 7.6440824E+13, 7.6291458E+13, 9.4645085E+13, 1.0124628E+14, 1.0354111E+14, + 1.0999650E+14, 1.0889946E+14, 1.1381912E+14, 1.3490042E+14, 1.5941519E+14, + 1.4983265E+14, 1.5184267E+14, 1.5991420E+14, 1.6976697E+14, 1.8771840E+14, + 1.6434367E+14, 1.8371960E+14, 2.1966369E+14, 1.9617879E+14, 2.2399700E+14, + 1.8429912E+14, 2.0129736E+14, 2.0541588E+14, 2.4334962E+14, 3.5077122E+14, + 3.4517894E+14, 3.5749668E+14, 3.6624304E+14, 3.4975113E+14, 3.5566025E+14, + 4.2825273E+14, 4.8406375E+14, 4.9511159E+14, 5.2695368E+14, 5.2401611E+14, + 5.0877746E+14, 4.8780853E+14}; + return etfphot_data; +} + +// This version uses scream_scorpio_interface to read netcdf files. +mam4::mo_photo::PhotoTableData read_photo_table( + const std::string &rsf_file, const std::string &xs_long_file) { + // set up the lng_indexer and pht_alias_mult_1 views based on our + // (hardwired) chemical mechanism + HostViewInt1D lng_indexer_h("lng_indexer", phtcnt); + + int nw, nump, numsza, numcolo3, numalb, nt, np_xs; // table dimensions + scorpio::register_file(rsf_file, scorpio::Read); + // read and broadcast dimension data + nump = scorpio::get_dimlen(rsf_file, "numz"); + numsza = scorpio::get_dimlen(rsf_file, "numsza"); + numalb = scorpio::get_dimlen(rsf_file, "numalb"); + numcolo3 = scorpio::get_dimlen(rsf_file, "numcolo3fact"); + + scorpio::register_file(xs_long_file, scorpio::Read); + nt = scorpio::get_dimlen(xs_long_file, "numtemp"); + nw = scorpio::get_dimlen(xs_long_file, "numwl"); + np_xs = scorpio::get_dimlen(xs_long_file, "numprs"); + + // FIXME: hard-coded for only one photo reaction. + std::string rxt_names[1] = {"jh2o2"}; + int numj = 1; + lng_indexer_h(0) = 0; + // allocate the photolysis table + auto table = mam4::mo_photo::create_photo_table_data( + nw, nt, np_xs, numj, nump, numsza, numcolo3, numalb); + + // allocate host views for table data + auto rsf_tab_h = Kokkos::create_mirror_view(table.rsf_tab); + auto xsqy_h = Kokkos::create_mirror_view(table.xsqy); + auto sza_h = Kokkos::create_mirror_view(table.sza); + auto alb_h = Kokkos::create_mirror_view(table.alb); + auto press_h = Kokkos::create_mirror_view(table.press); + auto colo3_h = Kokkos::create_mirror_view(table.colo3); + auto o3rat_h = Kokkos::create_mirror_view(table.o3rat); + // auto etfphot_h = Kokkos::create_mirror_view(table.etfphot); + auto prs_h = Kokkos::create_mirror_view(table.prs); + + // read file data into our host views + scorpio::read_var(rsf_file, "pm", press_h.data()); + scorpio::read_var(rsf_file, "sza", sza_h.data()); + scorpio::read_var(rsf_file, "alb", alb_h.data()); + scorpio::read_var(rsf_file, "colo3fact", o3rat_h.data()); + scorpio::read_var(rsf_file, "colo3", colo3_h.data()); + // it produces an error. + scorpio::read_var(rsf_file, "RSF", rsf_tab_h.data()); + scorpio::read_var(xs_long_file, "pressure", prs_h.data()); + + // read xsqy data (using lng_indexer_h for the first index) + // FIXME: hard-coded for only one photo reaction. + for(int m = 0; m < phtcnt; ++m) { + auto xsqy_ndx_h = ekat::subview(xsqy_h, m); + scorpio::read_var(xs_long_file, rxt_names[m], xsqy_h.data()); + } + + // populate etfphot by rebinning solar data + HostView1D wc_h("wc", nw), wlintv_h("wlintv", nw), we_h("we", nw + 1); + + scorpio::read_var(rsf_file, "wc", wc_h.data()); + scorpio::read_var(rsf_file, "wlintv", wlintv_h.data()); + for(int i = 0; i < nw; ++i) { + we_h(i) = wc_h(i) - 0.5 * wlintv_h(i); + } + we_h(nw) = wc_h(nw - 1) - 0.5 * wlintv_h(nw - 1); + // populate_etfphot(we_h, etfphot_h); + // FIXME: etfphot_data is hard-coded. + auto etfphot_data = populate_etfphot_from_e3sm_case(); + auto etfphot_h = HostView1D((Real *)etfphot_data.data(), nw); + + scorpio::release_file(rsf_file); + scorpio::release_file(xs_long_file); + + // copy host photolysis table into place on device + Kokkos::deep_copy(table.rsf_tab, rsf_tab_h); + Kokkos::deep_copy(table.xsqy, xsqy_h); + Kokkos::deep_copy(table.sza, sza_h); + Kokkos::deep_copy(table.alb, alb_h); + Kokkos::deep_copy(table.press, press_h); + Kokkos::deep_copy(table.colo3, colo3_h); + Kokkos::deep_copy(table.o3rat, o3rat_h); + Kokkos::deep_copy(table.etfphot, etfphot_h); + Kokkos::deep_copy(table.prs, prs_h); + // set pht_alias_mult_1 to 1 + Kokkos::deep_copy(table.pht_alias_mult_1, 1.0); + Kokkos::deep_copy(table.lng_indexer, lng_indexer_h); + + // compute gradients (on device) + Kokkos::parallel_for( + "del_p", nump - 1, KOKKOS_LAMBDA(int i) { + table.del_p(i) = 1.0 / haero::abs(table.press(i) - table.press(i + 1)); + }); + Kokkos::parallel_for( + "del_sza", numsza - 1, KOKKOS_LAMBDA(int i) { + table.del_sza(i) = 1.0 / (table.sza(i + 1) - table.sza(i)); + }); + Kokkos::parallel_for( + "del_alb", numalb - 1, KOKKOS_LAMBDA(int i) { + table.del_alb(i) = 1.0 / (table.alb(i + 1) - table.alb(i)); + }); + Kokkos::parallel_for( + "del_o3rat", numcolo3 - 1, KOKKOS_LAMBDA(int i) { + table.del_o3rat(i) = 1.0 / (table.o3rat(i + 1) - table.o3rat(i)); + }); + Kokkos::parallel_for( + "dprs", np_xs - 1, KOKKOS_LAMBDA(int i) { + table.dprs(i) = 1.0 / (table.prs(i) - table.prs(i + 1)); + }); + + return table; +} + +} // namespace scream::impl diff --git a/components/eamxx/src/physics/mam/readfiles/soil_erodibility.hpp b/components/eamxx/src/physics/mam/readfiles/soil_erodibility.hpp new file mode 100644 index 000000000000..8b47c81d9073 --- /dev/null +++ b/components/eamxx/src/physics/mam/readfiles/soil_erodibility.hpp @@ -0,0 +1,44 @@ +#ifndef SOIL_ERODIBILITY_HPP +#define SOIL_ERODIBILITY_HPP + +// For AtmosphereInput +#include "share/io/scorpio_input.hpp" + +namespace scream { +namespace soil_erodibility { + +template +struct soilErodibilityFunctions { + using Device = DeviceType; + + using KT = KokkosTypes; + using const_view_1d = typename KT::template view_1d; + + static std::shared_ptr create_horiz_remapper( + const std::shared_ptr &model_grid, + const std::string &soilErodibility_data_file, const std::string &map_file, + const std::string &field_name, const std::string &dim_name1); + + static std::shared_ptr create_data_reader( + const std::shared_ptr &horiz_remapper, + const std::string &data_file); + + static void update_soil_erodibility_data_from_file( + std::shared_ptr &scorpio_reader, + AbstractRemapper &horiz_interp, const_view_1d &input); + + static void init_soil_erodibility_file_read( + const int ncol, const std::string field_name, const std::string dim_name1, + const std::shared_ptr &grid, + const std::string &data_file, const std::string &mapping_file, + // output + std::shared_ptr &SoilErodibilityHorizInterp, + std::shared_ptr &SoilErodibilityDataReader); + +}; // struct soilErodilityFunctions + +} // namespace soil_erodibility +} // namespace scream +#endif // SOIL_ERODIBILITY_HPP + +#include "soil_erodibility_impl.hpp" \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/readfiles/soil_erodibility_impl.hpp b/components/eamxx/src/physics/mam/readfiles/soil_erodibility_impl.hpp new file mode 100644 index 000000000000..af0c4d73c174 --- /dev/null +++ b/components/eamxx/src/physics/mam/readfiles/soil_erodibility_impl.hpp @@ -0,0 +1,147 @@ +#ifndef SOIL_ERODIBILITY_IMPL_HPP +#define SOIL_ERODIBILITY_IMPL_HPP + +#include "share/grid/remap/identity_remapper.hpp" +#include "share/grid/remap/refining_remapper_p2p.hpp" +#include "share/io/scream_scorpio_interface.hpp" +#include "share/util/scream_timing.hpp" + +namespace scream { +namespace soil_erodibility { + +template +std::shared_ptr +soilErodibilityFunctions::create_horiz_remapper( + const std::shared_ptr &model_grid, + const std::string &data_file, const std::string &map_file, + const std::string &field_name, const std::string &dim_name1) { + using namespace ShortFieldTagsNames; + + scorpio::register_file(data_file, scorpio::Read); + const int ncols_data = scorpio::get_dimlen(data_file, dim_name1); + + scorpio::release_file(data_file); + + // We could use model_grid directly if using same num levels, + // but since shallow clones are cheap, we may as well do it (less lines of + // code) + auto horiz_interp_tgt_grid = + model_grid->clone("soil_erodibility_horiz_interp_tgt_grid", true); + + const int ncols_model = model_grid->get_num_global_dofs(); + std::shared_ptr remapper; + if(ncols_data == ncols_model) { + remapper = std::make_shared( + horiz_interp_tgt_grid, IdentityRemapper::SrcAliasTgt); + } else { + EKAT_REQUIRE_MSG(ncols_data <= ncols_model, + "Error! We do not allow to coarsen soil erodibility " + "data to fit the model. We only allow\n" + " soil erodibility data to be at the same or " + "coarser resolution as the model.\n"); + // We must have a valid map file + EKAT_REQUIRE_MSG(map_file != "", + "ERROR: soil erodibility data is on a different grid " + "than the model one,\n" + " but remap file is missing from soil erodibility " + "parameter list."); + + remapper = + std::make_shared(horiz_interp_tgt_grid, map_file); + } + + remapper->registration_begins(); + + const auto tgt_grid = remapper->get_tgt_grid(); + + const auto layout_2d = tgt_grid->get_2d_scalar_layout(); + const auto nondim = ekat::units::Units::nondimensional(); + + Field soil_erodibility( + FieldIdentifier(field_name, layout_2d, nondim, tgt_grid->name())); + soil_erodibility.allocate_view(); + + remapper->register_field_from_tgt(soil_erodibility); + + remapper->registration_ends(); + + return remapper; + +} // create_horiz_remapper + +// ------------------------------------------------------------------------------------------- +template +std::shared_ptr +soilErodibilityFunctions::create_data_reader( + const std::shared_ptr &horiz_remapper, + const std::string &data_file) { + std::vector io_fields; + for(int i = 0; i < horiz_remapper->get_num_fields(); ++i) { + io_fields.push_back(horiz_remapper->get_src_field(i)); + } + const auto io_grid = horiz_remapper->get_src_grid(); + return std::make_shared(data_file, io_grid, io_fields, true); +} // create_data_reader + +// ------------------------------------------------------------------------------------------- +template +void soilErodibilityFunctions::update_soil_erodibility_data_from_file( + std::shared_ptr &scorpio_reader, + AbstractRemapper &horiz_interp, const_view_1d &input) { + start_timer("EAMxx::soilErodibility::update_soil_erodibility_data_from_file"); + + // 1. Read from file + start_timer( + "EAMxx::soilErodibility::update_soil_erodibility_data_from_file::read_" + "data"); + scorpio_reader->read_variables(); + stop_timer( + "EAMxx::soilErodibility::update_soil_erodibility_data_from_file::read_" + "data"); + + // 2. Run the horiz remapper (it is a do-nothing op if soilErodibility data is + // on same grid as model) + start_timer( + "EAMxx::soilErodibility::update_soil_erodibility_data_from_file::horiz_" + "remap"); + horiz_interp.remap(/*forward = */ true); + stop_timer( + "EAMxx::soilErodibility::update_soil_erodibility_data_from_file::horiz_" + "remap"); + + // 3. Get the tgt field of the remapper + start_timer( + "EAMxx::soilErodibility::update_soil_erodibility_data_from_file::get_" + "field"); + // Recall, the fields are registered in the order: + // Read the field from the file + input = horiz_interp.get_tgt_field(0).get_view(); + stop_timer( + "EAMxx::soilErodibility::update_soil_erodibility_data_from_file::get_" + "field"); + + stop_timer("EAMxx::soilErodibility::update_soil_erodibility_data_from_file"); + +} // END update_soil_erodibility_data_from_file + +// ------------------------------------------------------------------------------------------- +template +void soilErodibilityFunctions::init_soil_erodibility_file_read( + const int ncol, const std::string field_name, const std::string dim_name1, + const std::shared_ptr &grid, + const std::string &data_file, const std::string &mapping_file, + // output + std::shared_ptr &soilErodibilityHorizInterp, + std::shared_ptr &soilErodibilityDataReader) { + // Init horizontal remap + soilErodibilityHorizInterp = create_horiz_remapper( + grid, data_file, mapping_file, field_name, dim_name1); + + // Create reader (an AtmosphereInput object) + soilErodibilityDataReader = + create_data_reader(soilErodibilityHorizInterp, data_file); +} // init_soil_erodibility_file_read +} // namespace soil_erodibility +} // namespace scream + +#endif // SOIL_ERODIBILITY_IMPL_HPP diff --git a/components/eamxx/src/physics/mam/readfiles/tracer_reader_utils.hpp b/components/eamxx/src/physics/mam/readfiles/tracer_reader_utils.hpp new file mode 100644 index 000000000000..6e45750354d5 --- /dev/null +++ b/components/eamxx/src/physics/mam/readfiles/tracer_reader_utils.hpp @@ -0,0 +1,758 @@ +#ifndef EAMXX_MAM_TRACER_READER_UTILS +#define EAMXX_MAM_TRACER_READER_UTILS + +#include +#include + +#include "share/grid/point_grid.hpp" +#include "share/grid/remap/coarsening_remapper.hpp" +#include "share/grid/remap/identity_remapper.hpp" +#include "share/grid/remap/refining_remapper_p2p.hpp" +#include "share/io/scorpio_input.hpp" +#include "share/io/scream_scorpio_interface.hpp" + +namespace scream::mam_coupling { + +using namespace ShortFieldTagsNames; +using view_1d_host = typename KT::view_1d::HostMirror; +using view_2d_host = typename KT::view_2d::HostMirror; + +using ExeSpace = typename KT::ExeSpace; +using ESU = ekat::ExeSpaceUtils; +using C = scream::physics::Constants; +using LIV = ekat::LinInterp; + +// Linoz NetCDF files use levs instead of formula_terms. +// This function allocates a view, so we need to do it during initialization. +// Thus, we assume that source pressure is independent of time, +// which is the case for Linoz files (zonal file). + +inline void compute_p_src_zonal_files(const view_1d &levs, + const view_2d &p_src) { + EKAT_REQUIRE_MSG(p_src.data() != 0, + "Error: p_src has not been allocated. \n"); + EKAT_REQUIRE_MSG(levs.data() != 0, "Error: levs has not been allocated. \n"); + const int ncol = p_src.extent(0); + const int nlevs_data = levs.extent(0); + EKAT_REQUIRE_MSG( + p_src.extent_int(1) == nlevs_data, + "Error: p_src has a different number of levels than the source data. \n"); + Kokkos::parallel_for( + "pressure_computation", + Kokkos::MDRangePolicy >({0, 0}, {ncol, nlevs_data}), + KOKKOS_LAMBDA(const int icol, const int kk) { + // mbar->pascals + // FIXME: Does EAMxx have a better method to + // convert units?" + p_src(icol, kk) = levs(kk) * 100; + }); + Kokkos::fence(); +} + +// We have a similar version in MAM4xx. +// This version was created because the data view cannot be modified +// inside the parallel_for. +// This struct will be used in init while reading nc files. +// The MAM4xx version will be used instead of parallel_for that loops over cols. +struct ForcingHelper { + // This index is in Fortran format. i.e. starts in 1 + int frc_ndx; + // does this file have altitude? + bool file_alt_data; + // number of sectors per forcing + int nsectors; + // offset in output vector from reader + int offset; +}; + +enum TracerFileType { + // file with ncol, lev, ilev, time and has P0 and PS as variables + // example: oxidants + FORMULA_PS, + // file with ncol, lev, ilev, time + // example: linoz + ZONAL, + // file with ncol, altitude, altitude_int, time + // example: elevated (at a height) emissions of aerosols and precursors + // NOTE: we must rename the default vertical tags when horiz remapping + // NOTE: we vert remap in a different routine in mam4xx + ELEVATED_EMISSIONS, + // Placeholder for cases where no file type is applicable + NONE +}; + +enum TracerDataIndex { BEG = 0, END = 1, OUT = 2 }; + +/* Maximum number of tracers (or fields) that the tracer reader can handle. + Note: We are not allocating memory for MAX_NVARS_TRACER tracers. + Therefore, if a file contains more than this number, it is acceptable to + increase this limit. Currently, Linoz files have 8 fields. */ +constexpr int MAX_NVARS_TRACER = 10; +constexpr int MAX_NUM_ELEVATED_EMISSIONS_FIELDS = 25; + +// Linoz structures to help manage all of the variables: +struct TracerTimeState { + // Whether the timestate has been initialized. + // The current month + int current_month = -1; + // Julian Date for the beginning of the month, as defined in + // /src/share/util/scream_time_stamp.hpp + // See this file for definition of Julian Date. + Real t_beg_month; + // Current simulation Julian Date + Real t_now; + // Number of days in the current month, cast as a Real + Real days_this_month; +}; // TricerTimeState + +struct TracerData { + TracerData() = default; + TracerData(const int ncol, const int nlev, const int nvars) { + init(ncol, nlev, nvars); + } + void init(const int ncol, const int nlev, const int nvars) { + ncol_ = ncol; + nlev_ = nlev; + nvars_ = nvars; + EKAT_REQUIRE_MSG( + nvars_ <= int(MAX_NVARS_TRACER), + "Error! Number of variables is bigger than NVARS_MAXTRACER. \n"); + } + + int ncol_{-1}; + int nlev_{-1}; + int nvars_{-1}; + int nlevs_data; + int ncols_data; + + // + int offset_time_index_{0}; + + // We cannot use a std::vector + // because we need to access these views from device. + // 0: beg 1: end 3: out + view_2d data[3][MAX_NVARS_TRACER]; + // type of file + TracerFileType file_type; + + // These views are employed in files with the PS variable + // 0: beg 1: end 3: out + view_1d ps[3]; + const_view_1d hyam; + const_view_1d hybm; + view_int_1d work_vert_inter[MAX_NVARS_TRACER]; + + // External forcing file (vertical emission) + // Uses altitude instead of pressure to interpolate data + view_1d altitude_int_; + // + bool has_altitude_{false}; + // pressure source for ZONAL or FORMULA_PS files + view_2d p_src_; + + // only for zonal files + view_1d zonal_levs_; + + void allocate_temporal_views() { + // BEG and OUT data views. + EKAT_REQUIRE_MSG(ncol_ != int(-1), "Error! ncols has not been set. \n"); + EKAT_REQUIRE_MSG(nlev_ != int(-1), "Error! nlevs has not been set. \n"); + EKAT_REQUIRE_MSG(nvars_ != int(-1), "Error! nvars has not been set. \n"); + + for(int ivar = 0; ivar < nvars_; ++ivar) { + data[TracerDataIndex::OUT][ivar] = view_2d("linoz_1_out", ncol_, nlev_); + data[TracerDataIndex::BEG][ivar] = view_2d("linoz_1_out", ncol_, nlev_); + } + + // for vertical interpolation using rebin routine + if(file_type == FORMULA_PS || file_type == ZONAL) { + // we only need work array for FORMULA_PS or ZONAL + for(int ivar = 0; ivar < nvars_; ++ivar) { + work_vert_inter[ivar] = + view_int_1d("allocate_work_vertical_interpolation", ncol_); + } + // we use ncremap and python scripts to convert zonal files to ne4pn4 + // grids. + p_src_ = view_2d("pressure_src_invariant", ncol_, nlev_); + } + + if(file_type == TracerFileType::FORMULA_PS) { + ps[TracerDataIndex::OUT] = view_1d("ps", ncol_); + ps[TracerDataIndex::BEG] = view_1d("ps", ncol_); + } + + if(file_type == TracerFileType::ZONAL) { + // we use ncremap and python scripts to convert zonal files to ne4pn4 + // grids. + compute_p_src_zonal_files(zonal_levs_, p_src_); + } + } +}; + +KOKKOS_INLINE_FUNCTION +Real linear_interp(const Real &x0, const Real &x1, const Real &t) { + return (1 - t) * x0 + t * x1; +} // linear_interp + +// time[3]={year,month, day} +inline util::TimeStamp convert_date(const int date) { + constexpr int ten_thousand = 10000; + constexpr int one_hundred = 100; + + int year = date / ten_thousand; + int month = (date - year * ten_thousand) / one_hundred; + int day = date - year * ten_thousand - month * one_hundred; + return util::TimeStamp(year, month, day, 0, 0, 0); +} +// FIXME: This function is not implemented in eamxx. +// FIXME: Assumes 365 days/year, 30 days/month; +// NOTE: that this assumption is mainly used for plotting. +// NOTE: This is not a direct port from EAM. +//We only use this routine for chlorine. +inline int compute_number_days_from_zero(const util::TimeStamp &ts) { + return ts.get_year() * 365 + ts.get_month() * 30 + ts.get_day(); +} + +inline void create_linoz_chlorine_reader( + const std::string &linoz_chlorine_file, const util::TimeStamp &model_time, + const int chlorine_loading_ymd, // in format YYYYMMDD + std::vector &values, std::vector &time_secs) { + auto time_stamp_beg = convert_date(chlorine_loading_ymd); + + const int offset_time = + compute_number_days_from_zero(time_stamp_beg) - compute_number_days_from_zero(model_time); + scorpio::register_file(linoz_chlorine_file, scorpio::Read); + const int nlevs_time = scorpio::get_time_len(linoz_chlorine_file); + for(int itime = 0; itime < nlevs_time; ++itime) { + int date; + scorpio::read_var(linoz_chlorine_file, "date", &date, itime); + if(date >= chlorine_loading_ymd) { + Real value; + scorpio::read_var(linoz_chlorine_file, "chlorine_loading", &value, itime); + values.push_back(value); + auto time_stamp = convert_date(date); + time_secs.push_back(compute_number_days_from_zero(time_stamp) - offset_time); + } + } // end itime + scorpio::release_file(linoz_chlorine_file); +} + +// Gets the times from the NC file +// Given a date in the format YYYYMMDD, returns its index in the time dimension. +inline void get_time_from_ncfile(const std::string &file_name, + const int cyclical_ymd, // in format YYYYMMDD + int &cyclical_ymd_index, + std::vector &dates) { + // in file_name: name of the NC file + // in cyclical_ymd: date in the format YYYYMMDD + // out cyclical_ymd_index: time index for cyclical_ymd + // out dates: date in YYYYMMDD format + scorpio::register_file(file_name, scorpio::Read); + const int nlevs_time = scorpio::get_time_len(file_name); + cyclical_ymd_index = -1; + for(int itime = 0; itime < nlevs_time; ++itime) { + int date; + scorpio::read_var(file_name, "date", &date, itime); + // std::cout << itime << " date: " << date << "\n"; + if(date >= cyclical_ymd && cyclical_ymd_index == -1) { + cyclical_ymd_index = itime; + } + dates.push_back(date); + } // end itime + + EKAT_REQUIRE_MSG(cyclical_ymd_index >= 0, + "Error! Current model time (" + + std::to_string(cyclical_ymd) + ") is not within " + + "Tracer time period: [" + std::to_string(dates[0]) + + ", " + "(" + std::to_string(dates[nlevs_time - 1]) + + ").\n"); + scorpio::release_file(file_name); +} + +inline Real chlorine_loading_advance(const util::TimeStamp &ts, + std::vector &values, + std::vector &time_secs) { + const int current_time = compute_number_days_from_zero(ts); + int index = 0; + // update index + for(int i = 0; i < int(values.size()); i++) { + if(current_time > time_secs[i]) { + index = i; + break; + } + } // + + const Real delt = (current_time - time_secs[index]) / + (time_secs[index + 1] - time_secs[index]); + return values[index] + delt * (values[index + 1] - values[index]); +} + +// It reads variables that are not time-dependent and independent of columns (no +// MPI involved here). We also obtain the offset_time_index using a date +// (cyclical_ymd) as input. We initialize a few members of tracer_data. +inline void setup_tracer_data(TracerData &tracer_data, // out + const std::string &trace_data_file, // in + const int cyclical_ymd) // in +{ + scorpio::register_file(trace_data_file, scorpio::Read); + // by default, I am assuming a zonal file. + TracerFileType tracer_file_type = ZONAL; + + int nlevs_data = -1; + if(scorpio::has_var(trace_data_file, "lev")) { + nlevs_data = scorpio::get_dimlen(trace_data_file, "lev"); + } + const bool has_altitude = scorpio::has_var(trace_data_file, "altitude"); + + // This type of files use altitude (zi) for vertical interpolation + if(has_altitude) { + nlevs_data = scorpio::get_dimlen(trace_data_file, "altitude"); + tracer_file_type = ELEVATED_EMISSIONS; + } + EKAT_REQUIRE_MSG( + nlevs_data != -1, + "Error: The file does not contain either lev or altitude. \n"); + + const int ncols_data = scorpio::get_dimlen(trace_data_file, "ncol"); + + // This type of files use model pressure (pmid) for vertical interpolation + if(scorpio::has_var(trace_data_file, "PS")) { + tracer_file_type = FORMULA_PS; + view_1d_host hyam_h("hyam_h", nlevs_data); + view_1d_host hybm_h("hybm_h", nlevs_data); + + scorpio::read_var(trace_data_file, "hyam", hyam_h.data()); + scorpio::read_var(trace_data_file, "hybm", hybm_h.data()); + view_1d hyam("hyam", nlevs_data); + view_1d hybm("hybm", nlevs_data); + Kokkos::deep_copy(hyam, hyam_h); + Kokkos::deep_copy(hybm, hybm_h); + tracer_data.hyam = hyam; + tracer_data.hybm = hybm; + } + + if(tracer_file_type == ZONAL) { + view_1d_host levs_h("levs_h", nlevs_data); + view_1d levs("levs", nlevs_data); + scorpio::read_var(trace_data_file, "lev", levs_h.data()); + Kokkos::deep_copy(levs, levs_h); + tracer_data.zonal_levs_ = levs; + } + + if(tracer_file_type == ELEVATED_EMISSIONS) { + const int nilevs_data = + scorpio::get_dimlen(trace_data_file, "altitude_int"); + view_1d_host altitude_int_host("altitude_int_host", nilevs_data); + view_1d altitude_int = view_1d("altitude_int", nilevs_data); + scorpio::read_var(trace_data_file, "altitude_int", + altitude_int_host.data()); + Kokkos::deep_copy(altitude_int, altitude_int_host); + tracer_data.altitude_int_ = altitude_int; + } + // time index + { + const int nlevs_time = scorpio::get_dimlen(trace_data_file, "time"); + int cyclical_ymd_index = -1; + for(int itime = 0; itime < nlevs_time; ++itime) { + int date; + scorpio::read_var(trace_data_file, "date", &date, itime); + if(date >= cyclical_ymd) { + cyclical_ymd_index = itime; + break; + } + } // end itime + + EKAT_REQUIRE_MSG(cyclical_ymd_index >= 0, "Error! Current model time (" + + std::to_string(cyclical_ymd) + + ") is not within " + + "Tracer time period.\n"); + + tracer_data.offset_time_index_ = cyclical_ymd_index; + } + + scorpio::release_file(trace_data_file); + tracer_data.file_type = tracer_file_type; + tracer_data.nlevs_data = nlevs_data; + tracer_data.ncols_data = ncols_data; + tracer_data.has_altitude_ = has_altitude; +} +inline std::shared_ptr create_horiz_remapper( + const std::shared_ptr &model_grid, + const std::string &trace_data_file, const std::string &map_file, + const std::vector &var_names, TracerData &tracer_data) { + using namespace ShortFieldTagsNames; + // We could use model_grid directly if using same num levels, + // but since shallow clones are cheap, we may as well do it (less lines of + // code) + auto horiz_interp_tgt_grid = + model_grid->clone("tracer_horiz_interp_tgt_grid", true); + horiz_interp_tgt_grid->reset_num_vertical_lev(tracer_data.nlevs_data); + + const int ncols_model = model_grid->get_num_global_dofs(); + std::shared_ptr remapper; + if(tracer_data.ncols_data == ncols_model) { + remapper = std::make_shared( + horiz_interp_tgt_grid, IdentityRemapper::SrcAliasTgt); + } else { + EKAT_REQUIRE_MSG(tracer_data.ncols_data <= ncols_model, + "Error! We do not allow to coarsen tracer external " + "forcing data to fit the " + "model. We only allow\n" + " tracer external forcing data to be at the same or " + "coarser resolution " + "as the model.\n"); + // We must have a valid map file + EKAT_REQUIRE_MSG( + map_file != "", + "ERROR: tracer external forcing data is on a different grid than the " + "model one,\n" + " but tracer external forcing data remap file is missing from " + "tracer external forcing data parameter list."); + + remapper = + std::make_shared(horiz_interp_tgt_grid, map_file); + } + + remapper->registration_begins(); + const auto tgt_grid = remapper->get_tgt_grid(); + const auto layout_2d = tgt_grid->get_2d_scalar_layout(); + + const auto layout_3d_mid = tgt_grid->get_3d_scalar_layout(true); + + const auto nondim = ekat::units::Units::nondimensional(); + + for(auto var_name : var_names) { + Field ifield( + FieldIdentifier(var_name, layout_3d_mid, nondim, tgt_grid->name())); + ifield.allocate_view(); + remapper->register_field_from_tgt(ifield); + } + // zonal files do not have the PS variable. + if(tracer_data.file_type == FORMULA_PS) { + Field ps(FieldIdentifier("PS", layout_2d, nondim, tgt_grid->name())); + ps.allocate_view(); + remapper->register_field_from_tgt(ps); + } + remapper->registration_ends(); + return remapper; + +} // create_horiz_remapper + +inline std::shared_ptr create_tracer_data_reader( + const std::shared_ptr &horiz_remapper, + const std::string &tracer_data_file, + const TracerFileType file_type = NONE) +{ + std::vector io_fields; + for(int i = 0; i < horiz_remapper->get_num_fields(); ++i) { + io_fields.push_back(horiz_remapper->get_src_field(i)); + } + const auto io_grid = horiz_remapper->get_src_grid(); + if(file_type == ELEVATED_EMISSIONS ){ + // NOTE: If we are using a vertical emission nc file with altitude instead of lev, + // we must rename this tag. + // We need to perform a shallow clone of io_grid because tags are const in this object. + auto horiz_interp_src_grid = + io_grid->clone("tracer_horiz_interp_src_grid", true); + horiz_interp_src_grid->reset_field_tag_name(LEV, "altitude"); + horiz_interp_src_grid->reset_field_tag_name(ILEV, "altitude_int"); + return std::make_shared(tracer_data_file, horiz_interp_src_grid, io_fields, + true); + } else{ + // We do not need to rename tags in or clone io_grid for other types of files. + return std::make_shared(tracer_data_file, io_grid, io_fields, + true); + } + + +} // create_tracer_data_reader + +inline void update_tracer_data_from_file( + const std::shared_ptr &scorpio_reader, + const int time_index, // zero-based + AbstractRemapper &tracer_horiz_interp, TracerData &tracer_data) { + // 1. read from field + scorpio_reader->read_variables(time_index); + // 2. Run the horiz remapper (it is a do-nothing op if tracer external forcing + // data is on same grid as model) + tracer_horiz_interp.remap(/*forward = */ true); + // + const int nvars = tracer_data.nvars_; + // + for(int i = 0; i < nvars; ++i) { + tracer_data.data[TracerDataIndex::END][i] = + tracer_horiz_interp.get_tgt_field(i).get_view(); + } + + if(tracer_data.file_type == FORMULA_PS) { + // Recall, the fields are registered in the order: tracers, ps + // 3. Copy from the tgt field of the remapper into the spa_data + tracer_data.ps[TracerDataIndex::END] = + tracer_horiz_interp.get_tgt_field(nvars).get_view(); + } + +} // update_tracer_data_from_file +inline void update_tracer_timestate( + const std::shared_ptr &scorpio_reader, + const util::TimeStamp &ts, AbstractRemapper &tracer_horiz_interp, + TracerTimeState &time_state, TracerData &data_tracer) { + // Now we check if we have to update the data that changes monthly + // NOTE: This means that tracer external forcing assumes monthly data to + // update. Not + // any other frequency. + const auto month = ts.get_month() - 1; // Make it 0-based + if(month != time_state.current_month) { + const auto tracer_data = data_tracer.data; + const int nvars = data_tracer.nvars_; + const auto ps = data_tracer.ps; + + // Update the tracer external forcing time state information + time_state.current_month = month; + time_state.t_beg_month = ts.curr_month_beg().frac_of_year_in_days(); + time_state.days_this_month = ts.days_in_curr_month(); + + // Copy spa_end'data into spa_beg'data, and read in the new spa_end + for(int ivar = 0; ivar < nvars; ++ivar) { + Kokkos::deep_copy(tracer_data[TracerDataIndex::BEG][ivar], + tracer_data[TracerDataIndex::END][ivar]); + } + + if(data_tracer.file_type == FORMULA_PS) { + Kokkos::deep_copy(ps[TracerDataIndex::BEG], ps[TracerDataIndex::END]); + } + + // Following SPA to time-interpolate data in MAM4xx + // Assume the data is saved monthly and cycles in one year + // Add offset_time_index to support cases where data is saved + // from other periods of time. + // Update the tracer external forcing data for this month and next month + // Start by copying next months data to this months data structure. + // NOTE: If the timestep is bigger than monthly this could cause the wrong + // values + // to be assigned. A timestep greater than a month is very unlikely + // so we will proceed. + int next_month = + data_tracer.offset_time_index_ + (time_state.current_month + 1) % 12; + update_tracer_data_from_file(scorpio_reader, next_month, + tracer_horiz_interp, data_tracer); + } + +} // END update_tracer_timestate + +// This function is based on the SPA::perform_time_interpolation function. +inline void perform_time_interpolation(const TracerTimeState &time_state, + const TracerData &data_tracer) { + // NOTE: we *assume* data_beg and data_end have the *same* hybrid v coords. + // IF this ever ceases to be the case, you can interp those too. + // Gather time stamp info + auto &t_now = time_state.t_now; + auto &t_beg = time_state.t_beg_month; + auto &delta_t = time_state.days_this_month; + + // We can ||ize over columns as well as over variables and bands + const auto& data = data_tracer.data; + const auto& file_type = data_tracer.file_type; + + const auto& ps = data_tracer.ps; + const int num_vars = data_tracer.nvars_; + + const int ncol = data_tracer.ncol_; + const int num_vert = data_tracer.nlev_; + + const int outer_iters = ncol * num_vars; + + const auto policy = ESU::get_default_team_policy(outer_iters, num_vert); + + auto delta_t_fraction = (t_now - t_beg) / delta_t; + + EKAT_REQUIRE_MSG( + delta_t_fraction >= 0 && delta_t_fraction <= 1, + "Error! Convex interpolation with coefficient out of [0,1].\n" + " t_now : " + + std::to_string(t_now) + + "\n" + " t_beg : " + + std::to_string(t_beg) + + "\n" + " delta_t: " + + std::to_string(delta_t) + "\n"); + + Kokkos::parallel_for( + "linoz_time_interp_loop", policy, KOKKOS_LAMBDA(const Team &team) { + // The policy is over ncols*num_vars, so retrieve icol/ivar + const int icol = team.league_rank() / num_vars; + const int ivar = team.league_rank() % num_vars; + + // Get column of beg/end/out variable + auto var_beg = ekat::subview(data[TracerDataIndex::BEG][ivar], icol); + auto var_end = ekat::subview(data[TracerDataIndex::END][ivar], icol); + auto var_out = ekat::subview(data[TracerDataIndex::OUT][ivar], icol); + + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, num_vert), [&](const int &k) { + var_out(k) = + linear_interp(var_beg(k), var_end(k), delta_t_fraction); + }); + // linoz files do not have ps variables. + if(ivar == 1 && file_type == FORMULA_PS) { + ps[TracerDataIndex::OUT](icol) = + linear_interp(ps[TracerDataIndex::BEG](icol), + ps[TracerDataIndex::END](icol), delta_t_fraction); + } + }); + Kokkos::fence(); +} // perform_time_interpolation + +inline void compute_source_pressure_levels(const view_1d &ps_src, + const view_2d &p_src, + const const_view_1d &hyam, + const const_view_1d &hybm) { + constexpr auto P0 = C::P0; + const int ncols = ps_src.extent(0); + const int num_vert_packs = p_src.extent(1); + const auto policy = ESU::get_default_team_policy(ncols, num_vert_packs); + + Kokkos::parallel_for( + "tracer_compute_p_src_loop", policy, KOKKOS_LAMBDA(const Team &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, num_vert_packs), [&](const int k) { + p_src(icol, k) = ps_src(icol) * hybm(k) + P0 * hyam(k); + }); + }); +} // compute_source_pressure_levels + +inline void perform_vertical_interpolation(const view_2d &p_src_c, + const const_view_2d &p_tgt_c, + const TracerData &input, + const view_2d output[]) { + const int ncol = input.ncol_; + const int levsiz = input.nlev_; + const int pver = mam4::nlev; + + const int num_vars = input.nvars_; + // make a local copy of output + view_2d output_local[MAX_NVARS_TRACER]; + EKAT_REQUIRE_MSG( + num_vars <= int(MAX_NVARS_TRACER), + "Error! Number of variables is bigger than NVARS_MAXTRACER. \n"); + for (int ivar = 0; ivar < num_vars; ++ivar) + { + // At this stage, begin/end must have the same horiz dimensions + EKAT_REQUIRE(input.ncol_ == output[ivar].extent_int(0)); + output_local[ivar] = output[ivar]; + } + const int outer_iters = ncol * num_vars; + const auto policy_setup = ESU::get_default_team_policy(outer_iters, pver); + const auto& data = input.data; + + Kokkos::parallel_for( + "vert_interp", policy_setup, + KOKKOS_LAMBDA(typename LIV::MemberType const &team) { + // The policy is over ncols*num_vars, so retrieve icol/ivar + const int icol = team.league_rank() / num_vars; + const int ivar = team.league_rank() % num_vars; + const auto pin_at_icol = ekat::subview(p_src_c, icol); + const auto pmid_at_icol = ekat::subview(p_tgt_c, icol); + const auto& datain = data[TracerDataIndex::OUT][ivar]; + const auto datain_at_icol = ekat::subview(datain, icol); + const auto dataout = output_local[ivar]; + const auto dataout_at_icol = ekat::subview(dataout, icol); + + mam4::vertical_interpolation::vert_interp( + team, levsiz, pver, pin_at_icol, pmid_at_icol, datain_at_icol, + dataout_at_icol); + }); +} + +inline void perform_vertical_interpolation(const const_view_1d &altitude_int, + const const_view_2d &zi, + const TracerData &input, + const view_2d output[]) { + EKAT_REQUIRE_MSG( + input.file_type == ELEVATED_EMISSIONS, + "Error! vertical interpolation only with altitude variable. \n"); + const int ncols = input.ncol_; + const int num_vars = input.nvars_; + const int num_vertical_lev_target = output[0].extent(1); + const int num_vert_packs = num_vertical_lev_target; + const int outer_iters = ncols * num_vars; + const auto policy_interp = + ESU::get_default_team_policy(outer_iters, num_vert_packs); + // FIXME: Get m2km from emaxx. + const Real m2km = 1e-3; + const auto &src_x = altitude_int; + const int nsrc = input.nlev_; + constexpr int pver = mam4::nlev; + const int pverp = pver + 1; + const auto& data = input.data; + + // make a local copy of output + view_2d output_local[MAX_NVARS_TRACER]; + EKAT_REQUIRE_MSG( + num_vars <= int(MAX_NVARS_TRACER), + "Error! Number of variables is bigger than NVARS_MAXTRACER. \n"); + for (int ivar = 0; ivar < num_vars; ++ivar) + { + // At this stage, begin/end must have the same horiz dimensions + EKAT_REQUIRE(input.ncol_ == output[ivar].extent_int(0)); + output_local[ivar] = output[ivar]; + } + Kokkos::parallel_for( + "tracer_vert_interp_loop", policy_interp, + KOKKOS_LAMBDA(const Team &team) { + const int icol = team.league_rank() / num_vars; + const int ivar = team.league_rank() % num_vars; + + const auto src = ekat::subview(data[TracerDataIndex::OUT][ivar], icol); + const auto trg = ekat::subview(output_local[ivar], icol); + // FIXME: Try to avoid copy of trg_x by modifying rebin + // trg_x + Real trg_x[pver + 1]; + // I am trying to do this: + // model_z(1:pverp) = m2km * state(c)%zi(i,pverp:1:-1) + for(int i = 0; i < pverp; ++i) { + trg_x[pverp - i - 1] = m2km * zi(icol, i); + } + team.team_barrier(); + mam4::vertical_interpolation::rebin(team, nsrc, num_vertical_lev_target, + src_x, trg_x, src, trg); + }); +} + +inline void advance_tracer_data( + const std::shared_ptr &scorpio_reader, // in + AbstractRemapper &tracer_horiz_interp, // out + const util::TimeStamp &ts, // in + TracerTimeState &time_state, TracerData &data_tracer, // out + const const_view_2d &p_tgt, const const_view_2d &zi_tgt, // in + const view_2d output[]) { // out + /* Update the TracerTimeState to reflect the current time, note the addition + * of dt */ + time_state.t_now = ts.frac_of_year_in_days(); + /* Update time state and if the month has changed, update the data.*/ + update_tracer_timestate(scorpio_reader, ts, tracer_horiz_interp, time_state, + data_tracer); + // Step 1. Perform time interpolation + perform_time_interpolation(time_state, data_tracer); + + if(data_tracer.file_type == FORMULA_PS) { + // Step 2. Compute source pressure levels + const auto ps = data_tracer.ps[TracerDataIndex::OUT]; + compute_source_pressure_levels(ps, data_tracer.p_src_, data_tracer.hyam, + data_tracer.hybm); + } + + // Step 3. Perform vertical interpolation + if(data_tracer.file_type == FORMULA_PS || data_tracer.file_type == ZONAL) { + perform_vertical_interpolation(data_tracer.p_src_, p_tgt, data_tracer, + output); + } else if(data_tracer.file_type == ELEVATED_EMISSIONS) { + perform_vertical_interpolation(data_tracer.altitude_int_, zi_tgt, + data_tracer, output); + } + +} // advance_tracer_data + +} // namespace scream::mam_coupling +#endif // EAMXX_MAM_HELPER_MICRO diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 29aaca421ea9..8dc5be1d05a6 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -4,8 +4,6 @@ #include "share/util/scream_timing.hpp" namespace scream::mam_coupling { -namespace { - template struct srfEmissFunctions { using Device = DeviceType; @@ -131,7 +129,6 @@ struct srfEmissFunctions { std::shared_ptr &SrfEmissDataReader); }; // struct srfEmissFunctions -} // namespace } // namespace scream::mam_coupling #endif // SRF_EMISSION_HPP diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 64ae65714c59..e1c471edf4be 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -6,8 +6,6 @@ #include "share/io/scream_scorpio_interface.hpp" namespace scream::mam_coupling { -namespace { - template std::shared_ptr srfEmissFunctions::create_horiz_remapper( @@ -104,9 +102,6 @@ void srfEmissFunctions::perform_time_interpolation( // NOTE: we *assume* data_beg and data_end have the *same* hybrid v coords. // IF this ever ceases to be the case, you can interp those too. - using ExeSpace = typename KT::ExeSpace; - using ESU = ekat::ExeSpaceUtils; - // Gather time stamp info auto &t_now = time_state.t_now; auto &t_beg = time_state.t_beg_month; @@ -184,8 +179,6 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( const int time_index, // zero-based AbstractRemapper &srfEmiss_horiz_interp, srfEmissInput &srfEmiss_input) { using namespace ShortFieldTagsNames; - using ESU = ekat::ExeSpaceUtils; - using Member = typename KokkosTypes::MemberType; start_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file"); @@ -206,13 +199,6 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( // Recall, the fields are registered in the order: ps, ccn3, g_sw, ssa_sw, // tau_sw, tau_lw - const auto &layout = srfEmiss_horiz_interp.get_tgt_field(0) - .get_header() - .get_identifier() - .get_layout(); - - const int ncols = layout.dim(COL); - // Read fields from the file for(int i = 0; i < srfEmiss_horiz_interp.get_num_fields(); ++i) { auto sector = @@ -241,10 +227,8 @@ void srfEmissFunctions::update_srfEmiss_timestate( if(month != time_state.current_month) { // Update the srfEmiss time state information time_state.current_month = month; - time_state.t_beg_month = - util::TimeStamp({ts.get_year(), month + 1, 1}, {0, 0, 0}) - .frac_of_year_in_days(); - time_state.days_this_month = util::days_in_month(ts.get_year(), month + 1); + time_state.t_beg_month = ts.curr_month_beg().frac_of_year_in_days(); + time_state.days_this_month = ts.days_in_curr_month(); // Copy srfEmiss_end'data into srfEmiss_beg'data, and read in the new // srfEmiss_end @@ -286,8 +270,6 @@ void srfEmissFunctions::init_srf_emiss_objects( SrfEmissDataReader = create_srfEmiss_data_reader(SrfEmissHorizInterp, data_file); } // init_srf_emiss_objects - -} // namespace } // namespace scream::mam_coupling #endif // SRF_EMISSION_IMPL_HPP diff --git a/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp b/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp index c423cb25cee9..ce3b64b3dc8a 100644 --- a/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp +++ b/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp @@ -50,7 +50,7 @@ void MLCorrection::set_grids( add_field("sfc_flux_sw_net", scalar2d, W/m2, grid_name); add_field("sfc_flux_lw_dn", scalar2d, W/m2, grid_name); m_lat = m_grid->get_geometry_data("lat"); - m_lon = m_grid->get_geometry_data("lon"); + m_lon = m_grid->get_geometry_data("lon"); } /* ----------------------- WARNING --------------------------------*/ @@ -60,20 +60,20 @@ void MLCorrection::set_grids( * to be used here which we can then setup using the m_fields_ml_output_variables variable */ add_field("T_mid", scalar3d_mid, K, grid_name, ps); - add_field("qv", scalar3d_mid, kg/kg, grid_name, "tracers", ps); add_field("horiz_winds", vector3d_mid, m/s, grid_name, ps); /* Note: we also need to update the precipitation after ML commits any column drying */ add_field("pseudo_density", scalar3d_mid, Pa, grid_name, ps); add_field("precip_liq_surf_mass", scalar2d, kg/m2, grid_name); add_field("precip_ice_surf_mass", scalar2d, kg/m2, grid_name); /* ----------------------- WARNING --------------------------------*/ + add_tracer("qv", m_grid, kg/kg, ps); add_group("tracers", grid_name, 1, Bundling::Required); } // ========================================================================================= void MLCorrection::initialize_impl(const RunType /* run_type */) { fpe_mask = ekat::get_enabled_fpes(); - ekat::disable_all_fpes(); // required for importing numpy + ekat::disable_all_fpes(); // required for importing numpy if ( Py_IsInitialized() == 0 ) { pybind11::initialize_interpreter(); } @@ -103,7 +103,7 @@ void MLCorrection::run_impl(const double dt) { std::string datetime_str = current_ts.get_date_string() + " " + current_ts.get_time_string(); const auto &phis = get_field_in("phis").get_view(); - const auto &sfc_alb_dif_vis = get_field_in("sfc_alb_dif_vis").get_view(); + const auto &sfc_alb_dif_vis = get_field_in("sfc_alb_dif_vis").get_view(); const auto &qv = get_field_out("qv").get_view(); const auto &T_mid = get_field_out("T_mid").get_view(); @@ -135,29 +135,29 @@ void MLCorrection::run_impl(const double dt) { pybind11::array_t( m_num_cols * m_num_levs, T_mid.data(), pybind11::str{}), pybind11::array_t( - m_num_cols * m_num_levs * num_tracers, qv.data(), pybind11::str{}), + m_num_cols * m_num_levs * num_tracers, qv.data(), pybind11::str{}), pybind11::array_t( - m_num_cols * m_num_levs, u.data(), pybind11::str{}), + m_num_cols * m_num_levs, u.data(), pybind11::str{}), pybind11::array_t( - m_num_cols * m_num_levs, v.data(), pybind11::str{}), + m_num_cols * m_num_levs, v.data(), pybind11::str{}), pybind11::array_t( - m_num_cols, h_lat.data(), pybind11::str{}), + m_num_cols, h_lat.data(), pybind11::str{}), pybind11::array_t( m_num_cols, h_lon.data(), pybind11::str{}), pybind11::array_t( - m_num_cols, phis.data(), pybind11::str{}), + m_num_cols, phis.data(), pybind11::str{}), pybind11::array_t( m_num_cols * (m_num_levs+1), SW_flux_dn.data(), pybind11::str{}), pybind11::array_t( m_num_cols, sfc_alb_dif_vis.data(), pybind11::str{}), pybind11::array_t( - m_num_cols, sfc_flux_sw_net.data(), pybind11::str{}), + m_num_cols, sfc_flux_sw_net.data(), pybind11::str{}), pybind11::array_t( - m_num_cols, sfc_flux_lw_dn.data(), pybind11::str{}), - m_num_cols, m_num_levs, num_tracers, dt, + m_num_cols, sfc_flux_lw_dn.data(), pybind11::str{}), + m_num_cols, m_num_levs, num_tracers, dt, ML_model_tq, ML_model_uv, ML_model_sfc_fluxes, datetime_str); - pybind11::gil_scoped_release no_gil; - ekat::enable_fpes(fpe_mask); + pybind11::gil_scoped_release no_gil; + ekat::enable_fpes(fpe_mask); // Now back out the qv change abd apply it to precipitation, only if Tq ML is turned on if (m_ML_model_path_tq != "None") { @@ -171,7 +171,7 @@ void MLCorrection::run_impl(const double dt) { constexpr Real g = PC::gravit; const auto num_levs = m_num_levs; const auto policy = ESU::get_default_team_policy(m_num_cols, m_num_levs); - + const auto &qv_told = qv_in.get_view(); const auto &qv_tnew = get_field_in("qv").get_view(); Kokkos::parallel_for("Compute WVP diff", policy, @@ -233,4 +233,4 @@ void MLCorrection::finalize_impl() { } // ========================================================================================= -} // namespace scream +} // namespace scream diff --git a/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp b/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp index 8ebfc9c14c01..069f4ec49150 100644 --- a/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp +++ b/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp @@ -47,7 +47,7 @@ Nudging::Nudging (const ekat::Comm& comm, const ekat::ParameterList& params) "Error! Inconsistent 'lev' dimension found in nudging data files."); } // use nudging weights - if (m_use_weights) + if (m_use_weights) m_weights_file = m_params.get("nudging_weights_file"); // TODO: Add some warning messages here. @@ -84,7 +84,7 @@ void Nudging::set_grids(const std::shared_ptr grids_manager) add_field("T_mid", scalar3d_layout_mid, K, grid_name, ps); } if (ekat::contains(m_fields_nudge,"qv")) { - add_field("qv", scalar3d_layout_mid, kg/kg, grid_name, "tracers", ps); + add_tracer("qv", m_grid, kg/kg, ps); } if (ekat::contains(m_fields_nudge,"U") or ekat::contains(m_fields_nudge,"V")) { add_field("horiz_winds", horiz_wind_layout, m/s, grid_name, ps); @@ -107,7 +107,7 @@ void Nudging::set_grids(const std::shared_ptr grids_manager) /* Check for consistency between nudging files, map file, and remapper */ // Number of columns globally - auto num_cols_global = m_grid->get_num_global_dofs(); + auto num_cols_global = m_grid->get_num_global_dofs(); // Get the information from the first nudging data file int num_cols_src = scorpio::get_dimlen(m_datafiles[0],"ncol"); @@ -136,7 +136,7 @@ void Nudging::set_grids(const std::shared_ptr grids_manager) << "model grid and/or the mapfile."); EKAT_REQUIRE_MSG(m_use_weights == false, "Error! Nudging::set_grids - it seems that the user intends to use both nuding " - << "from coarse data as well as weighted nudging simultaneously. This is not supported. " + << "from coarse data as well as weighted nudging simultaneously. This is not supported. " << "If the user wants to use both at their own risk, the user should edit the source code " << "by deleting this error message."); // If we get here, we are good to go! diff --git a/components/eamxx/src/physics/nudging/tests/nudging_tests_helpers.hpp b/components/eamxx/src/physics/nudging/tests/nudging_tests_helpers.hpp index ea3b1b2f238b..ab0a8420ade0 100644 --- a/components/eamxx/src/physics/nudging/tests/nudging_tests_helpers.hpp +++ b/components/eamxx/src/physics/nudging/tests/nudging_tests_helpers.hpp @@ -150,7 +150,6 @@ create_om (const std::string& filename_prefix, params.set("Averaging Type","INSTANT"); params.set("filename_prefix",filename_prefix); params.set("Floating Point Precision","real"); - params.set("MPI Ranks in Filename", false); params.set("Field Names",strvec_t{"p_mid","U","V"}); params.set("fill_value",fill_val); @@ -160,7 +159,8 @@ create_om (const std::string& filename_prefix, ctrl_pl.set("save_grid_data",false); auto om = std::make_shared(); - om->setup(comm,params,fm,gm,t0,t0,false); + om->initialize(comm,params,t0,false); + om->setup(fm,gm); return om; } diff --git a/components/eamxx/src/physics/p3/CMakeLists.txt b/components/eamxx/src/physics/p3/CMakeLists.txt index a1dfc946267d..6c39a9fcf1a6 100644 --- a/components/eamxx/src/physics/p3/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/CMakeLists.txt @@ -1,19 +1,8 @@ set(P3_SRCS - p3_f90.cpp - p3_ic_cases.cpp - p3_iso_c.f90 - ${SCREAM_BASE_DIR}/../eam/src/physics/p3/scream/micro_p3.F90 eamxx_p3_process_interface.cpp eamxx_p3_run.cpp ) -if (NOT SCREAM_LIB_ONLY) - list(APPEND P3_SRCS - p3_functions_f90.cpp - p3_main_wrap.cpp - ) # Add f90 bridges needed for testing -endif() - # Add ETI source files if not on CUDA/HIP if (NOT EAMXX_ENABLE_GPU OR Kokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE OR Kokkos_ENABLE_HIP_RELOCATABLE_DEVICE_CODE) list(APPEND P3_SRCS @@ -27,6 +16,7 @@ if (NOT EAMXX_ENABLE_GPU OR Kokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE OR Kokkos eti/p3_table_ice.cpp eti/p3_dsd2.cpp eti/p3_find.cpp + eti/p3_init.cpp eti/p3_update_prognostics.cpp eti/p3_get_time_space_phys_variables.cpp eti/p3_autoconversion.cpp @@ -77,6 +67,8 @@ if (SCREAM_P3_SMALL_KERNELS) add_library(p3 ${P3_SRCS} ${P3_SK_SRCS}) else() add_library(p3 ${P3_SRCS}) + # If small kernels are ON, we don't need a separate executable to test them. + # Also, we never want to generate baselines with this separate executable if (NOT SCREAM_LIBS_ONLY AND NOT SCREAM_ONLY_GENERATE_BASELINES) add_library(p3_sk ${P3_SRCS} ${P3_SK_SRCS}) # Always build p3_sk with SCREAM_P3_SMALL_KERNELS on @@ -88,11 +80,7 @@ endif() target_compile_definitions(p3 PUBLIC EAMXX_HAS_P3) foreach (P3_LIB IN LISTS P3_LIBS) - set_target_properties(${P3_LIB} PROPERTIES - Fortran_MODULE_DIRECTORY ${P3_LIB}/modules - ) target_include_directories(${P3_LIB} PUBLIC - ${CMAKE_CURRENT_BINARY_DIR}/${P3_LIB}/modules ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/impl ${SCREAM_BASE_DIR}/../eam/src/physics/cam @@ -108,10 +96,10 @@ else() endif() set (P3_TABLES scream/tables/p3_lookup_table_1.dat-v4.1.1 - scream/tables/mu_r_table_vals.dat${PRECISION_SUFFIX} - scream/tables/revap_table_vals.dat${PRECISION_SUFFIX} - scream/tables/vm_table_vals.dat${PRECISION_SUFFIX} - scream/tables/vn_table_vals.dat${PRECISION_SUFFIX} + scream/tables/mu_r_table_vals_v2.dat${PRECISION_SUFFIX} + scream/tables/revap_table_vals_v2.dat${PRECISION_SUFFIX} + scream/tables/vm_table_vals_v2.dat${PRECISION_SUFFIX} + scream/tables/vn_table_vals_v2.dat${PRECISION_SUFFIX} ) include (ScreamUtils) @@ -119,15 +107,6 @@ foreach (file IN ITEMS ${P3_TABLES}) GetInputFile(${file}) endforeach() -# This executable can be used to re-generate tables in ${SCREAM_DATA_DIR} -add_executable(p3_tables_setup EXCLUDE_FROM_ALL p3_tables_setup.cpp) -target_link_libraries(p3_tables_setup p3) - -#crusher change -if (Kokkos_ENABLE_HIP) -set_source_files_properties(p3_functions_f90.cpp PROPERTIES COMPILE_FLAGS -O0) -endif() - if (NOT SCREAM_LIB_ONLY) add_subdirectory(tests) endif() diff --git a/components/eamxx/src/physics/p3/disp/p3_check_values_impl_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_check_values_impl_disp.cpp index cc13e99bc91d..5bac83064778 100644 --- a/components/eamxx/src/physics/p3/disp/p3_check_values_impl_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_check_values_impl_disp.cpp @@ -16,7 +16,7 @@ ::check_values_disp(const uview_2d& qv, const uview_2d using ExeSpace = typename KT::ExeSpace; const Int nk_pack = ekat::npack(nk); const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack); - + Kokkos::parallel_for( "p3_check_values", policy, KOKKOS_LAMBDA(const MemberType& team) { @@ -32,4 +32,3 @@ ::check_values_disp(const uview_2d& qv, const uview_2d } // namespace p3 } // namespace scream - diff --git a/components/eamxx/src/physics/p3/disp/p3_cloud_sed_impl_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_cloud_sed_impl_disp.cpp index 8b755be4857f..4b528663d236 100644 --- a/components/eamxx/src/physics/p3/disp/p3_cloud_sed_impl_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_cloud_sed_impl_disp.cpp @@ -48,7 +48,7 @@ ::cloud_sedimentation_disp( } cloud_sedimentation( - ekat::subview(qc_incld, i), ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(cld_frac_l, i), + ekat::subview(qc_incld, i), ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(cld_frac_l, i), ekat::subview(acn, i), ekat::subview(inv_dz, i), dnu, team, workspace, nk, ktop, kbot, kdir, dt, inv_dt, do_predict_nc, ekat::subview(qc, i), ekat::subview(nc, i), ekat::subview(nc_incld, i), ekat::subview(mu_c, i), ekat::subview(lamc, i), ekat::subview(qc_tend, i), @@ -60,4 +60,3 @@ ::cloud_sedimentation_disp( } // namespace p3 } // namespace scream - diff --git a/components/eamxx/src/physics/p3/disp/p3_ice_sed_impl_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_ice_sed_impl_disp.cpp index f1e84750c250..16e04fee6ab8 100644 --- a/components/eamxx/src/physics/p3/disp/p3_ice_sed_impl_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_ice_sed_impl_disp.cpp @@ -29,7 +29,7 @@ ::ice_sedimentation_disp( const uview_1d& precip_ice_surf, const uview_1d& nucleationPossible, const uview_1d& hydrometeorsPresent, - const physics::P3_Constants & p3constants) + const P3Runtime& runtime_options) { using ExeSpace = typename KT::ExeSpace; const Int nk_pack = ekat::npack(nk); @@ -50,7 +50,7 @@ ::ice_sedimentation_disp( ekat::subview(inv_dz, i), team, workspace, nk, ktop, kbot, kdir, dt, inv_dt, ekat::subview(qi, i), ekat::subview(qi_incld, i), ekat::subview(ni, i), ekat::subview(ni_incld, i), ekat::subview(qm, i), ekat::subview(qm_incld, i), ekat::subview(bm, i), ekat::subview(bm_incld, i), ekat::subview(qi_tend, i), ekat::subview(ni_tend, i), - ice_table_vals, precip_ice_surf(i), p3constants); + ice_table_vals, precip_ice_surf(i), runtime_options); }); } diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp index b79b4f6d072c..99a2381ca23d 100644 --- a/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp @@ -110,8 +110,7 @@ ::p3_main_internal_disp( const P3Temporaries& temporaries, const WorkspaceManager& workspace_mgr, Int nj, - Int nk, - const physics::P3_Constants & p3constants) + Int nk) { using ExeSpace = typename KT::ExeSpace; @@ -124,6 +123,8 @@ ::p3_main_internal_disp( const Int kbot = kdir == -1 ? nk-1 : 0; constexpr bool debug_ABORT = false; + const bool do_ice_production = runtime_options.do_ice_production; + // per-column bools view_1d nucleationPossible("nucleationPossible", nj); view_1d hydrometeorsPresent("hydrometeorsPresent", nj); @@ -244,7 +245,7 @@ ::p3_main_internal_disp( T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, qv, th, qc, nc, qr, nr, qi, ni, qm, bm, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, - ni_incld, bm_incld, nucleationPossible, hydrometeorsPresent, p3constants); + ni_incld, bm_incld, nucleationPossible, hydrometeorsPresent, runtime_options); // ------------------------------------------------------------------------------------------ // main k-loop (for processes): @@ -259,7 +260,7 @@ ::p3_main_internal_disp( nr_incld, ni_incld, bm_incld, mu_c, nu, lamc, cdist, cdist1, cdistr, mu_r, lamr, logn0r, qv2qi_depos_tend, precip_total_tend, nevapr, qr_evap_tend, vap_liq_exchange, vap_ice_exchange, liq_ice_exchange, - pratot, prctot, nucleationPossible, hydrometeorsPresent, p3constants); + pratot, prctot, nucleationPossible, hydrometeorsPresent, runtime_options); //NOTE: At this point, it is possible to have negative (but small) nc, nr, ni. This is not // a problem; those values get clipped to zero in the sedimentation section (if necessary). @@ -285,19 +286,21 @@ ::p3_main_internal_disp( rho, inv_rho, rhofacr, cld_frac_r, inv_dz, qr_incld, workspace_mgr, lookup_tables.vn_table_vals, lookup_tables.vm_table_vals, nj, nk, ktop, kbot, kdir, infrastructure.dt, inv_dt, qr, nr, nr_incld, mu_r, lamr, precip_liq_flux, qtend_ignore, ntend_ignore, - diagnostic_outputs.precip_liq_surf, nucleationPossible, hydrometeorsPresent, p3constants); + diagnostic_outputs.precip_liq_surf, nucleationPossible, hydrometeorsPresent, runtime_options); // Ice sedimentation: (adaptive substepping) ice_sedimentation_disp( rho, inv_rho, rhofaci, cld_frac_i, inv_dz, workspace_mgr, nj, nk, ktop, kbot, kdir, infrastructure.dt, inv_dt, qi, qi_incld, ni, ni_incld, qm, qm_incld, bm, bm_incld, qtend_ignore, ntend_ignore, - lookup_tables.ice_table_vals, diagnostic_outputs.precip_ice_surf, nucleationPossible, hydrometeorsPresent, p3constants); + lookup_tables.ice_table_vals, diagnostic_outputs.precip_ice_surf, nucleationPossible, hydrometeorsPresent, runtime_options); // homogeneous freezing f cloud and rain - homogeneous_freezing_disp( - T_atm, inv_exner, nj, nk, ktop, kbot, kdir, qc, nc, qr, nr, qi, - ni, qm, bm, th, nucleationPossible, hydrometeorsPresent); + if(do_ice_production) { + homogeneous_freezing_disp(T_atm, inv_exner, nj, nk, ktop, kbot, kdir, qc, + nc, qr, nr, qi, ni, qm, bm, th, + nucleationPossible, hydrometeorsPresent); + } // // final checks to ensure consistency of mass/number @@ -309,7 +312,7 @@ ::p3_main_internal_disp( qm, bm, mu_c, nu, lamc, mu_r, lamr, vap_liq_exchange, ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, rho_qi, diag_equiv_reflectivity, diag_eff_radius_qc, diag_eff_radius_qr, nucleationPossible, hydrometeorsPresent, - p3constants); + runtime_options); // // merge ice categories with similar properties @@ -335,4 +338,3 @@ ::p3_main_internal_disp( } // namespace p3 } // namespace scream - diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_part1_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_part1_disp.cpp index 28db7a6348dc..b2d3276dc067 100644 --- a/components/eamxx/src/physics/p3/disp/p3_main_impl_part1_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_part1_disp.cpp @@ -60,7 +60,7 @@ ::p3_main_part1_disp( const uview_2d& bm_incld, const uview_1d& nucleationPossible, const uview_1d& hydrometeorsPresent, - const physics::P3_Constants & p3constants) + const P3Runtime& runtime_options) { using ExeSpace = typename KT::ExeSpace; const Int nk_pack = ekat::npack(nk); @@ -81,7 +81,7 @@ ::p3_main_part1_disp( ekat::subview(qv, i), ekat::subview(th_atm, i), ekat::subview(qc, i), ekat::subview(nc, i), ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(qi, i), ekat::subview(ni, i), ekat::subview(qm, i), ekat::subview(bm, i), ekat::subview(qc_incld, i), ekat::subview(qr_incld, i), ekat::subview(qi_incld, i), ekat::subview(qm_incld, i), ekat::subview(nc_incld, i), ekat::subview(nr_incld, i), ekat::subview(ni_incld, i), ekat::subview(bm_incld, i), - nucleationPossible(i), hydrometeorsPresent(i), p3constants); + nucleationPossible(i), hydrometeorsPresent(i), runtime_options); }); } diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp index 91bb01989aef..9f7987abbb59 100644 --- a/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp @@ -90,7 +90,7 @@ ::p3_main_part2_disp( const uview_2d& prctot, const uview_1d& nucleationPossible, const uview_1d& hydrometeorsPresent, - const physics::P3_Constants & p3constants) + const P3Runtime& runtime_options) { using ExeSpace = typename KT::ExeSpace; const Int nk_pack = ekat::npack(nk); @@ -125,7 +125,7 @@ ::p3_main_part2_disp( ekat::subview(cdist1, i), ekat::subview(cdistr, i), ekat::subview(mu_r, i), ekat::subview(lamr, i), ekat::subview(logn0r, i), ekat::subview(qv2qi_depos_tend, i), ekat::subview(precip_total_tend, i), ekat::subview(nevapr, i), ekat::subview(qr_evap_tend, i), ekat::subview(vap_liq_exchange, i), ekat::subview(vap_ice_exchange, i), ekat::subview(liq_ice_exchange, i), - ekat::subview(pratot, i), ekat::subview(prctot, i), hydrometeorsPresent(i), nk, p3constants); + ekat::subview(pratot, i), ekat::subview(prctot, i), hydrometeorsPresent(i), nk, runtime_options); if (!hydrometeorsPresent(i)) return; }); diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_part3_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_part3_disp.cpp index fd41fb5f0fe1..10e7c911c5da 100644 --- a/components/eamxx/src/physics/p3/disp/p3_main_impl_part3_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_part3_disp.cpp @@ -55,7 +55,7 @@ ::p3_main_part3_disp( const uview_2d& diag_eff_radius_qr, const uview_1d& nucleationPossible, const uview_1d& hydrometeorsPresent, - const physics::P3_Constants & p3constants) + const P3Runtime& runtime_options) { using ExeSpace = typename KT::ExeSpace; const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack); @@ -81,7 +81,7 @@ ::p3_main_part3_disp( ekat::subview(mu_c, i), ekat::subview(nu, i), ekat::subview(lamc, i), ekat::subview(mu_r, i), ekat::subview(lamr, i), ekat::subview(vap_liq_exchange, i), ekat::subview(ze_rain, i), ekat::subview(ze_ice, i), ekat::subview(diag_vm_qi, i), ekat::subview(diag_eff_radius_qi, i), ekat::subview(diag_diam_qi, i), ekat::subview(rho_qi, i), ekat::subview(diag_equiv_reflectivity, i), ekat::subview(diag_eff_radius_qc, i), - ekat::subview(diag_eff_radius_qr, i), p3constants); + ekat::subview(diag_eff_radius_qr, i), runtime_options); }); } diff --git a/components/eamxx/src/physics/p3/disp/p3_rain_sed_impl_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_rain_sed_impl_disp.cpp index e77c91035174..a1b38afe8edd 100644 --- a/components/eamxx/src/physics/p3/disp/p3_rain_sed_impl_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_rain_sed_impl_disp.cpp @@ -28,7 +28,7 @@ ::rain_sedimentation_disp( const uview_1d& precip_liq_surf, const uview_1d& nucleationPossible, const uview_1d& hydrometeorsPresent, - const physics::P3_Constants & p3constants) + const P3Runtime& runtime_options) { using ExeSpace = typename KT::ExeSpace; const Int nk_pack = ekat::npack(nk); @@ -45,12 +45,12 @@ ::rain_sedimentation_disp( // Rain sedimentation: (adaptive substepping) rain_sedimentation( - ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(rhofacr, i), ekat::subview(cld_frac_r, i), - ekat::subview(inv_dz, i), ekat::subview(qr_incld, i), - team, workspace, vn_table_vals, vm_table_vals, nk, ktop, kbot, kdir, dt, inv_dt, - ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(nr_incld, i), ekat::subview(mu_r, i), - ekat::subview(lamr, i), ekat::subview(precip_liq_flux, i), - ekat::subview(qr_tend, i), ekat::subview(nr_tend, i), precip_liq_surf(i), p3constants); + ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(rhofacr, i), ekat::subview(cld_frac_r, i), + ekat::subview(inv_dz, i), ekat::subview(qr_incld, i), + team, workspace, vn_table_vals, vm_table_vals, nk, ktop, kbot, kdir, dt, inv_dt, + ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(nr_incld, i), ekat::subview(mu_r, i), + ekat::subview(lamr, i), ekat::subview(precip_liq_flux, i), + ekat::subview(qr_tend, i), ekat::subview(nr_tend, i), precip_liq_surf(i), runtime_options); }); } diff --git a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp index c414d7093f62..15437fd567a9 100644 --- a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp +++ b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp @@ -1,10 +1,7 @@ -#include "physics/p3/eamxx_p3_process_interface.hpp" #include "share/property_checks/field_within_interval_check.hpp" #include "share/property_checks/field_lower_bound_check.hpp" -// Needed for p3_init, the only F90 code still used. -#include "physics/p3/p3_functions.hpp" -#include "physics/share/physics_constants.hpp" -#include "physics/p3/p3_f90.hpp" +#include "p3_functions.hpp" +#include "eamxx_p3_process_interface.hpp" #include "ekat/ekat_assert.hpp" #include "ekat/util/ekat_units.hpp" @@ -72,15 +69,15 @@ void P3Microphysics::set_grids(const std::shared_ptr grids_m add_field ("T_mid", scalar3d_layout_mid, K, grid_name, ps); // T_mid is the only one of these variables that is also updated. // Prognostic State: (all fields are both input and output) - add_field("qv", scalar3d_layout_mid, kg/kg, grid_name, "tracers", ps); - add_field("qc", scalar3d_layout_mid, kg/kg, grid_name, "tracers", ps); - add_field("qr", scalar3d_layout_mid, kg/kg, grid_name, "tracers", ps); - add_field("qi", scalar3d_layout_mid, kg/kg, grid_name, "tracers", ps); - add_field("qm", scalar3d_layout_mid, kg/kg, grid_name, "tracers", ps); - add_field("nc", scalar3d_layout_mid, 1/kg, grid_name, "tracers", ps); - add_field("nr", scalar3d_layout_mid, 1/kg, grid_name, "tracers", ps); - add_field("ni", scalar3d_layout_mid, 1/kg, grid_name, "tracers", ps); - add_field("bm", scalar3d_layout_mid, 1/kg, grid_name, "tracers", ps); + add_tracer("qv", m_grid, kg/kg, ps); + add_tracer("qc", m_grid, kg/kg, ps); + add_tracer("qr", m_grid, kg/kg, ps); + add_tracer("qi", m_grid, kg/kg, ps); + add_tracer("qm", m_grid, kg/kg, ps); + add_tracer("nc", m_grid, 1/kg, ps); + add_tracer("nr", m_grid, 1/kg, ps); + add_tracer("ni", m_grid, 1/kg, ps); + add_tracer("bm", m_grid, 1/kg, ps); // Diagnostic Inputs: (only the X_prev fields are both input and output, all others are just inputs) add_field("nc_nuceat_tend", scalar3d_layout_mid, 1/(kg*s), grid_name, ps); @@ -220,13 +217,8 @@ void P3Microphysics::init_buffers(const ATMBufferManager &buffer_manager) // ========================================================================================= void P3Microphysics::initialize_impl (const RunType /* run_type */) { - // Gather runtime options - runtime_options.max_total_ni = m_params.get("max_total_ni"); - - // setting P3 constants in a struct - m_p3constants.set_p3_from_namelist(m_params); - m_p3constants.print_p3constants(m_atm_logger); - // done setting P3 constants + // Gather runtime options from file + runtime_options.load_runtime_options_from_file(m_params); // Set property checks for fields in this process add_invariant_check(get_field_out("T_mid"),m_grid,100.0,500.0,false); @@ -249,8 +241,8 @@ void P3Microphysics::initialize_impl (const RunType /* run_type */) add_postcondition_check(get_field_out("eff_radius_qr"),m_grid,0.0,5.0e3,false); // Initialize p3 - p3::p3_init(/* write_tables = */ false, - this->get_comm().am_i_root()); + lookup_tables = P3F::p3_init(/* write_tables = */ false, + this->get_comm().am_i_root()); // Initialize all of the structures that are passed to p3_main in run_impl. // Note: Some variables in the structures are not stored in the field manager. For these @@ -288,7 +280,7 @@ void P3Microphysics::initialize_impl (const RunType /* run_type */) p3_preproc.set_variables(m_num_cols,nk_pack,pmid,pmid_dry,pseudo_density,pseudo_density_dry, T_atm,cld_frac_t, qv, qc, nc, qr, nr, qi, qm, ni, bm, qv_prev, - inv_exner, th_atm, cld_frac_l, cld_frac_i, cld_frac_r, dz); + inv_exner, th_atm, cld_frac_l, cld_frac_i, cld_frac_r, dz, runtime_options); // --Prognostic State Variables: prog_state.qc = p3_preproc.qc; prog_state.nc = p3_preproc.nc; @@ -419,12 +411,6 @@ void P3Microphysics::initialize_impl (const RunType /* run_type */) p3_postproc.set_mass_and_energy_fluxes(vapor_flux, water_flux, ice_flux, heat_flux); } - // Load tables - P3F::init_kokkos_ice_lookup_tables(lookup_tables.ice_table_vals, lookup_tables.collect_table_vals); - P3F::init_kokkos_tables(lookup_tables.vn_table_vals, lookup_tables.vm_table_vals, - lookup_tables.revap_table_vals, lookup_tables.mu_r_table_vals, - lookup_tables.dnu_table_vals); - // Setup WSM for internal local variables const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(m_num_cols, nk_pack); workspace_mgr.setup(m_buffer.wsm_data, nk_pack_p1, 52, policy); diff --git a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.hpp b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.hpp index bd592db16162..c1323bdc8e8e 100644 --- a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.hpp +++ b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.hpp @@ -22,7 +22,6 @@ namespace scream class P3Microphysics : public AtmosphereProcess { using P3F = p3::Functions; - using CP3 = physics::P3_Constants; using Spack = typename P3F::Spack; using Smask = typename P3F::Smask; using Pack = ekat::Pack; @@ -54,8 +53,6 @@ class P3Microphysics : public AtmosphereProcess // Set the grid void set_grids (const std::shared_ptr grids_manager); - CP3 m_p3constants; - /*--------------------------------------------------------------------------------------------*/ // Most individual processes have a pre-processing step that constructs needed variables from // the set of fields stored in the field manager. A structure like this defines those operations, @@ -108,23 +105,26 @@ class P3Microphysics : public AtmosphereProcess th_atm(icol,ipack) = PF::calculate_theta_from_T(T_atm_pack,pmid_pack); // Cloud fraction // Set minimum cloud fraction - avoids division by zero - cld_frac_l(icol,ipack) = ekat::max(cld_frac_t_pack,mincld); - cld_frac_i(icol,ipack) = ekat::max(cld_frac_t_pack,mincld); - cld_frac_r(icol,ipack) = ekat::max(cld_frac_t_pack,mincld); + // Alternatively set fraction to 1 everywhere to disable subgrid effects + cld_frac_l(icol,ipack) = runtime_opts.set_cld_frac_l_to_one ? 1 : ekat::max(cld_frac_t_pack,mincld); + cld_frac_i(icol,ipack) = runtime_opts.set_cld_frac_i_to_one ? 1 : ekat::max(cld_frac_t_pack,mincld); + cld_frac_r(icol,ipack) = runtime_opts.set_cld_frac_r_to_one ? 1 : ekat::max(cld_frac_t_pack,mincld); // update rain cloud fraction given neighboring levels using max-overlap approach. - for (int ivec=0;iveccld_frac_r(icol,ipack)[ivec] ? - cld_frac_t(icol,ipack_m1)[ivec_m1] : - cld_frac_r(icol,ipack)[ivec]; + if ( !runtime_opts.set_cld_frac_r_to_one ) { + for (int ivec=0;iveccld_frac_r(icol,ipack)[ivec] ? + cld_frac_t(icol,ipack_m1)[ivec_m1] : + cld_frac_r(icol,ipack)[ivec]; + } } } // @@ -155,6 +155,8 @@ class P3Microphysics : public AtmosphereProcess view_2d cld_frac_i; view_2d cld_frac_r; view_2d dz; + // Add runtime_options as a member variable + P3F::P3Runtime runtime_opts; // Assigning local variables void set_variables(const int ncol, const int npack, const view_2d_const& pmid_, const view_2d_const& pmid_dry_, @@ -164,7 +166,8 @@ class P3Microphysics : public AtmosphereProcess const view_2d& nc_, const view_2d& qr_, const view_2d& nr_, const view_2d& qi_, const view_2d& qm_, const view_2d& ni_, const view_2d& bm_, const view_2d& qv_prev_, const view_2d& inv_exner_, const view_2d& th_atm_, const view_2d& cld_frac_l_, - const view_2d& cld_frac_i_, const view_2d& cld_frac_r_, const view_2d& dz_ + const view_2d& cld_frac_i_, const view_2d& cld_frac_r_, const view_2d& dz_, + const P3F::P3Runtime& runtime_options ) { m_ncol = ncol; @@ -193,6 +196,7 @@ class P3Microphysics : public AtmosphereProcess cld_frac_i = cld_frac_i_; cld_frac_r = cld_frac_r_; dz = dz_; + runtime_opts = runtime_options; } // set_variables }; // p3_preamble /* --------------------------------------------------------------------------------------------*/ diff --git a/components/eamxx/src/physics/p3/eamxx_p3_run.cpp b/components/eamxx/src/physics/p3/eamxx_p3_run.cpp index cdb156d71f62..3de8184319af 100644 --- a/components/eamxx/src/physics/p3/eamxx_p3_run.cpp +++ b/components/eamxx/src/physics/p3/eamxx_p3_run.cpp @@ -33,7 +33,7 @@ void P3Microphysics::run_impl (const double dt) #ifdef SCREAM_P3_SMALL_KERNELS temporaries, #endif - workspace_mgr, m_num_cols, m_num_levs, m_p3constants); + workspace_mgr, m_num_cols, m_num_levs); // Conduct the post-processing of the p3_main output. Kokkos::parallel_for( diff --git a/components/eamxx/src/physics/p3/eti/p3_init.cpp b/components/eamxx/src/physics/p3/eti/p3_init.cpp new file mode 100644 index 000000000000..b1878af89840 --- /dev/null +++ b/components/eamxx/src/physics/p3/eti/p3_init.cpp @@ -0,0 +1,14 @@ +#include "p3_init_impl.hpp" + +namespace scream { +namespace p3 { + +/* + * Explicit instantiation for doing find functions on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace p3 +} // namespace scream diff --git a/components/eamxx/src/physics/p3/impl/p3_autoconversion_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_autoconversion_impl.hpp index 43b1f21b7a74..7b521b7db58f 100644 --- a/components/eamxx/src/physics/p3/impl/p3_autoconversion_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_autoconversion_impl.hpp @@ -3,7 +3,6 @@ #include "p3_functions.hpp" // for ETI only but harmless for GPU #include "p3_subgrid_variance_scaling_impl.hpp" -#include "physics/share/physics_constants.hpp" namespace scream { namespace p3 { @@ -14,28 +13,43 @@ void Functions ::cloud_water_autoconversion( const Spack& rho, const Spack& qc_incld, const Spack& nc_incld, const Spack& inv_qc_relvar, Spack& qc2qr_autoconv_tend, Spack& nc2nr_autoconv_tend, Spack& ncautr, - const physics::P3_Constants & p3constants, + const P3Runtime& runtime_options, const Smask& context) { // Khroutdinov and Kogan (2000) const auto qc_not_small = qc_incld >= 1e-8 && context; - constexpr Scalar CONS3 = C::CONS3; - const Scalar p3_autoconversion_prefactor = p3constants.p3_autoconversion_prefactor; - -//printf(" hey inside AAAAAAAAAAAAAAAA %13.6f \n", p3_autoconversion_factor); + const Scalar autoconversion_prefactor = + runtime_options.autoconversion_prefactor; + const Scalar autoconversion_qc_exponent = + runtime_options.autoconversion_qc_exponent; + const Scalar autoconversion_nc_exponent = + runtime_options.autoconversion_nc_exponent; + const Scalar autoconversion_radius = runtime_options.autoconversion_radius; + + Scalar CONS3; + // TODO: always default to second branch after BFB stuff is addressed + if(autoconversion_radius == sp(25.0e-6)) { + CONS3 = C::CONS3; + } else { + CONS3 = sp(1.0) / (C::CONS2 * pow(autoconversion_radius, sp(3.0))); + } - if(qc_not_small.any()){ + if(qc_not_small.any()) { Spack sgs_var_coef; // sgs_var_coef = subgrid_variance_scaling(inv_qc_relvar, sp(2.47) ); sgs_var_coef = 1; - qc2qr_autoconv_tend.set(qc_not_small, - sgs_var_coef*p3_autoconversion_prefactor*pow(qc_incld,sp(2.47))*pow(nc_incld*sp(1.e-6)*rho,sp(-1.79))); + qc2qr_autoconv_tend.set( + qc_not_small, + sgs_var_coef * autoconversion_prefactor * + pow(qc_incld, autoconversion_qc_exponent) * + pow(nc_incld * sp(1.e-6) * rho, -autoconversion_nc_exponent)); // note: ncautr is change in Nr; nc2nr_autoconv_tend is change in Nc - ncautr.set(qc_not_small, qc2qr_autoconv_tend*CONS3); - nc2nr_autoconv_tend.set(qc_not_small, qc2qr_autoconv_tend*nc_incld/qc_incld); + ncautr.set(qc_not_small, qc2qr_autoconv_tend * CONS3); + nc2nr_autoconv_tend.set(qc_not_small, + qc2qr_autoconv_tend * nc_incld / qc_incld); } nc2nr_autoconv_tend.set(qc2qr_autoconv_tend == 0 && context, 0); diff --git a/components/eamxx/src/physics/p3/impl/p3_cldliq_imm_freezing_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_cldliq_imm_freezing_impl.hpp index 2abcc3ef135e..c23e9e00e634 100644 --- a/components/eamxx/src/physics/p3/impl/p3_cldliq_imm_freezing_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_cldliq_imm_freezing_impl.hpp @@ -20,7 +20,7 @@ ::cldliq_immersion_freezing( const Spack& mu_c, const Spack& cdist1, const Spack& qc_incld, const Spack& inv_qc_relvar, Spack& qc2qi_hetero_freeze_tend, Spack& nc2ni_immers_freeze_tend, - const physics::P3_Constants & p3constants, + const P3Runtime& runtime_options, const Smask& context) { constexpr Scalar qsmall = C::QSMALL; @@ -28,13 +28,15 @@ ::cldliq_immersion_freezing( constexpr Scalar T_zerodegc = C::T_zerodegc; constexpr Scalar CONS5 = C::CONS5; constexpr Scalar CONS6 = C::CONS6; - const Scalar p3_a_imm = p3constants.p3_a_imm; + const Scalar immersion_freezing_exponent = + runtime_options.immersion_freezing_exponent; const auto qc_not_small_and_t_freezing = (qc_incld >= qsmall) && (T_atm <= T_rainfrz) && context; if (qc_not_small_and_t_freezing.any()) { Spack expAimmDt, inv_lamc3; - expAimmDt.set(qc_not_small_and_t_freezing, exp(p3_a_imm * (T_zerodegc-T_atm))); + expAimmDt.set(qc_not_small_and_t_freezing, + exp(immersion_freezing_exponent * (T_zerodegc - T_atm))); inv_lamc3.set(qc_not_small_and_t_freezing, cube(1/lamc)); Spack sgs_var_coef; diff --git a/components/eamxx/src/physics/p3/impl/p3_cloud_rain_acc_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_cloud_rain_acc_impl.hpp index 5d64cb593afb..b65c45d22cf5 100644 --- a/components/eamxx/src/physics/p3/impl/p3_cloud_rain_acc_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_cloud_rain_acc_impl.hpp @@ -20,23 +20,35 @@ ::cloud_rain_accretion( const Spack& qc_incld, const Spack& nc_incld, const Spack& qr_incld, const Spack& inv_qc_relvar, Spack& qc2qr_accret_tend, Spack& nc_accret_tend, - const physics::P3_Constants & p3constants, + const P3Runtime& runtime_options, const Smask& context) { constexpr Scalar qsmall = C::QSMALL; - const Scalar p3_k_accretion = p3constants.p3_k_accretion;; + const Scalar accretion_prefactor = runtime_options.accretion_prefactor; + const Scalar accretion_qc_exponent = runtime_options.accretion_qc_exponent; + const Scalar accretion_qr_exponent = runtime_options.accretion_qr_exponent; Spack sgs_var_coef; // sgs_var_coef = subgrid_variance_scaling(inv_qc_relvar, sp(1.15) ); sgs_var_coef = 1; const auto qr_and_qc_not_small = (qr_incld >= qsmall) && (qc_incld >= qsmall) && context; - if (qr_and_qc_not_small.any()) { + if(qr_and_qc_not_small.any()) { // Khroutdinov and Kogan (2000) - qc2qr_accret_tend.set(qr_and_qc_not_small, - sgs_var_coef * sp(p3_k_accretion) * pow(qc_incld * qr_incld, sp(1.15))); - nc_accret_tend.set(qr_and_qc_not_small, qc2qr_accret_tend * nc_incld / qc_incld); + // TODO: always default to second branch after BFB stuff is addressed + if(accretion_qc_exponent == accretion_qr_exponent) { + qc2qr_accret_tend.set(qr_and_qc_not_small, + sgs_var_coef * accretion_prefactor * + pow(qc_incld * qr_incld, accretion_qr_exponent)); + } else { + qc2qr_accret_tend.set(qr_and_qc_not_small, + sgs_var_coef * accretion_prefactor * + pow(qc_incld, accretion_qc_exponent) * + pow(qr_incld, accretion_qr_exponent)); + } + nc_accret_tend.set(qr_and_qc_not_small, + qc2qr_accret_tend * nc_incld / qc_incld); qc2qr_accret_tend.set(nc_accret_tend == 0 && context, 0); nc_accret_tend.set(qc2qr_accret_tend == 0 && context, 0); diff --git a/components/eamxx/src/physics/p3/impl/p3_dsd2_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_dsd2_impl.hpp index c7c434c0daee..68814db34790 100644 --- a/components/eamxx/src/physics/p3/impl/p3_dsd2_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_dsd2_impl.hpp @@ -77,7 +77,7 @@ void Functions:: get_rain_dsd2 ( const Spack& qr, Spack& nr, Spack& mu_r, Spack& lamr, - const physics::P3_Constants & p3constants, + const P3Runtime& runtime_options, const Smask& context) { constexpr auto nsmall = C::NSMALL; @@ -88,7 +88,7 @@ get_rain_dsd2 ( const auto qr_gt_small = qr >= qsmall && context; - const Scalar mu_r_const = p3constants.p3_mu_r_constant; + const Scalar mu_r_const = runtime_options.constant_mu_rain; if (qr_gt_small.any()) { // use lookup table to get mu diff --git a/components/eamxx/src/physics/p3/impl/p3_ice_collection_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_ice_collection_impl.hpp index 2c3d5f6fc22c..d7250b2d9f2b 100644 --- a/components/eamxx/src/physics/p3/impl/p3_ice_collection_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_ice_collection_impl.hpp @@ -15,7 +15,7 @@ ::ice_cldliq_collection( const Spack& qi_incld, const Spack& qc_incld, const Spack& ni_incld, const Spack& nc_incld, Spack& qc2qi_collect_tend, Spack& nc_collect_tend, Spack& qc2qr_ice_shed_tend, Spack& ncshdc, - const physics::P3_Constants & p3constants, + const P3Runtime& runtime_options, const Smask& context) { constexpr Scalar qsmall = C::QSMALL; @@ -28,17 +28,25 @@ ::ice_cldliq_collection( const auto both_ge_small = qi_incld_ge_small && qc_incld_ge_small && context; const auto both_ge_small_pos_t = both_ge_small && !t_is_negative; - const Scalar p3_eci = p3constants.p3_eci; + const Scalar cldliq_to_ice_collection_factor = + runtime_options.cldliq_to_ice_collection_factor; constexpr auto inv_dropmass = C::ONE/C::dropmass; qc2qi_collect_tend.set(both_ge_small && t_is_negative, - rhofaci*table_val_qc2qi_collect*qc_incld*p3_eci*rho*ni_incld); - nc_collect_tend.set(both_ge_small, rhofaci*table_val_qc2qi_collect*nc_incld*p3_eci*rho*ni_incld); + rhofaci * table_val_qc2qi_collect * qc_incld * + cldliq_to_ice_collection_factor * rho * ni_incld); + nc_collect_tend.set(both_ge_small, + rhofaci * table_val_qc2qi_collect * nc_incld * + cldliq_to_ice_collection_factor * rho * ni_incld); // for T_atm > 273.15, assume cloud water is collected and shed as rain drops // sink for cloud water mass and number, note qcshed is source for rain mass - qc2qr_ice_shed_tend.set(both_ge_small_pos_t, rhofaci*table_val_qc2qi_collect*qc_incld*p3_eci*rho*ni_incld); - nc_collect_tend.set(both_ge_small_pos_t, rhofaci*table_val_qc2qi_collect*nc_incld*p3_eci*rho*ni_incld); + qc2qr_ice_shed_tend.set(both_ge_small_pos_t, + rhofaci * table_val_qc2qi_collect * qc_incld * + cldliq_to_ice_collection_factor * rho * ni_incld); + nc_collect_tend.set(both_ge_small_pos_t, + rhofaci * table_val_qc2qi_collect * nc_incld * + cldliq_to_ice_collection_factor * rho * ni_incld); // source for rain number, assume 1 mm drops are shed ncshdc.set(both_ge_small_pos_t, qc2qr_ice_shed_tend*inv_dropmass); } @@ -53,7 +61,7 @@ ::ice_rain_collection( const Spack& qi_incld, const Spack& ni_incld, const Spack& qr_incld, Spack& qr2qi_collect_tend, Spack& nr_collect_tend, - const physics::P3_Constants & p3constants, + const P3Runtime& runtime_options, const Smask& context) { constexpr Scalar qsmall = C::QSMALL; @@ -67,11 +75,17 @@ ::ice_rain_collection( const auto both_ge_small_neg_t = both_ge_small && t_is_negative; constexpr Scalar ten = 10.0; - const Scalar p3_eri = p3constants.p3_eri; + const Scalar rain_to_ice_collection_factor = + runtime_options.rain_to_ice_collection_factor; // note: table_val_qr2qi_collect and logn0r are already calculated as log_10 - qr2qi_collect_tend.set(both_ge_small_neg_t, pow(ten, table_val_qr2qi_collect+logn0r)*rho*rhofaci*p3_eri*ni_incld); - nr_collect_tend.set(both_ge_small_neg_t, pow(ten, table_val_nr_collect+logn0r)*rho*rhofaci*p3_eri*ni_incld); + qr2qi_collect_tend.set(both_ge_small_neg_t, + pow(ten, table_val_qr2qi_collect + logn0r) * rho * + rhofaci * rain_to_ice_collection_factor * + ni_incld); + nr_collect_tend.set(both_ge_small_neg_t, + pow(ten, table_val_nr_collect + logn0r) * rho * rhofaci * + rain_to_ice_collection_factor * ni_incld); // rain number sink due to collection // for T_atm > 273.15, assume collected rain number is shed as @@ -80,7 +94,8 @@ ::ice_rain_collection( // rate of ice mass due to melting // collection of rain above freezing does not impact total rain mass nr_collect_tend.set(both_ge_small && !t_is_negative, - pow(ten, table_val_nr_collect + logn0r)*rho*rhofaci*p3_eri*ni_incld); + pow(ten, table_val_nr_collect + logn0r) * rho * rhofaci * + rain_to_ice_collection_factor * ni_incld); // for now neglect shedding of ice collecting rain above freezing, since snow is // not expected to shed in these conditions (though more hevaily rimed ice would be // expected to lead to shedding) diff --git a/components/eamxx/src/physics/p3/impl/p3_ice_nucleation_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_ice_nucleation_impl.hpp index 27313934fa7e..cb8d961c523f 100644 --- a/components/eamxx/src/physics/p3/impl/p3_ice_nucleation_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_ice_nucleation_impl.hpp @@ -13,7 +13,7 @@ ::ice_nucleation( const Spack& temp, const Spack& inv_rho, const Spack& ni, const Spack& ni_activated, const Spack& qv_supersat_i, const Scalar& inv_dt, const bool& do_predict_nc, const bool& do_prescribed_CCN, Spack& qv2qi_nucleat_tend, Spack& ni_nucleat_tend, - const physics::P3_Constants & p3constants, + const P3Runtime& runtime_options, const Smask& context) { constexpr Scalar nsmall = C::NSMALL; @@ -23,7 +23,8 @@ ::ice_nucleation( constexpr Scalar piov3 = C::PIOV3; constexpr Scalar mi0 = sp(4.0)*piov3*sp(900.0)*sp(1.e-18); - const Scalar p3_dep_nucleation_exponent = p3constants.p3_dep_nucleation_exponent; + const Scalar deposition_nucleation_exponent = + runtime_options.deposition_nucleation_exponent; const auto t_lt_T_icenuc = temp < T_icenuc; const auto qv_supersat_i_ge_005 = qv_supersat_i >= 0.05; @@ -36,7 +37,7 @@ ::ice_nucleation( Spack dum{0.0}, N_nuc{0.0}, Q_nuc{0.0}; if (any_if_not_log.any()) { - dum = sp(0.005)*exp(sp(p3_dep_nucleation_exponent)*(tmelt-temp))*sp(1.0e3)*inv_rho; + dum = sp(0.005)*exp(sp(deposition_nucleation_exponent)*(tmelt-temp))*sp(1.0e3)*inv_rho; dum = min(dum, sp(1.0e5)*inv_rho); diff --git a/components/eamxx/src/physics/p3/impl/p3_ice_sed_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_ice_sed_impl.hpp index 1273d7bbc206..6be83a900aff 100644 --- a/components/eamxx/src/physics/p3/impl/p3_ice_sed_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_ice_sed_impl.hpp @@ -17,13 +17,13 @@ typename Functions::Spack Functions ::calc_bulk_rho_rime( const Spack& qi_tot, Spack& qi_rim, Spack& bi_rim, - const physics::P3_Constants & p3constants, + const P3Runtime& runtime_options, const Smask& context) { constexpr Scalar bsmall = C::BSMALL; constexpr Scalar qsmall = C::QSMALL; - const Scalar p3_rho_rime_min = p3constants.p3_rho_rime_min; - const Scalar p3_rho_rime_max = p3constants.p3_rho_rime_max; + const Scalar min_rime_rho = runtime_options.min_rime_rho; + const Scalar max_rime_rho = runtime_options.max_rime_rho; Spack rho_rime(0); @@ -33,12 +33,12 @@ ::calc_bulk_rho_rime( rho_rime.set(bi_rim_gt_small, qi_rim / bi_rim); } - Smask rho_rime_lt_min = rho_rime < p3_rho_rime_min; - Smask rho_rime_gt_max = rho_rime > p3_rho_rime_max; + Smask rho_rime_lt_min = rho_rime < min_rime_rho; + Smask rho_rime_gt_max = rho_rime > max_rime_rho; // impose limits on rho_rime; adjust bi_rim if needed - rho_rime.set(bi_rim_gt_small && rho_rime_lt_min, p3_rho_rime_min); - rho_rime.set(bi_rim_gt_small && rho_rime_gt_max, p3_rho_rime_max); + rho_rime.set(bi_rim_gt_small && rho_rime_lt_min, min_rime_rho); + rho_rime.set(bi_rim_gt_small && rho_rime_gt_max, max_rime_rho); Smask adjust = bi_rim_gt_small && (rho_rime_gt_max || rho_rime_lt_min); if (adjust.any()) { bi_rim.set(adjust, qi_rim / rho_rime); @@ -87,7 +87,7 @@ ::ice_sedimentation( const uview_1d& ni_tend, const view_ice_table& ice_table_vals, Scalar& precip_ice_surf, - const physics::P3_Constants & p3constants) + const P3Runtime& runtime_options) { // Get temporary workspaces needed for the ice-sed calculation uview_1d V_qit, V_nit, flux_nit, flux_bir, flux_qir, flux_qit; @@ -105,7 +105,7 @@ ::ice_sedimentation( constexpr Scalar qsmall = C::QSMALL; constexpr Scalar nsmall = C::NSMALL; - const Scalar p3_ice_sed_knob = p3constants.p3_ice_sed_knob; + const Scalar ice_sedimentation_factor = runtime_options.ice_sedimentation_factor; bool log_qxpresent; const Int k_qxtop = find_top(team, sqi, qsmall, kbot, ktop, kdir, log_qxpresent); @@ -145,7 +145,7 @@ ::ice_sedimentation( // impose lower limits to prevent log(<0) ni_incld(pk).set(qi_gt_small, max(ni_incld(pk), nsmall)); - const auto rhop = calc_bulk_rho_rime(qi_incld(pk), qm_incld(pk), bm_incld(pk), p3constants, qi_gt_small); + const auto rhop = calc_bulk_rho_rime(qi_incld(pk), qm_incld(pk), bm_incld(pk), runtime_options, qi_gt_small); qm(pk).set(qi_gt_small, qm_incld(pk)*cld_frac_i(pk) ); bm(pk).set(qi_gt_small, bm_incld(pk)*cld_frac_i(pk) ); @@ -163,8 +163,8 @@ ::ice_sedimentation( ni_incld(pk).set(qi_gt_small, max(ni_incld(pk), table_val_ni_lammin * ni_incld(pk))); ni(pk).set(qi_gt_small, ni_incld(pk) * cld_frac_i(pk)); - V_qit(pk).set(qi_gt_small, p3_ice_sed_knob * table_val_qi_fallspd * rhofaci(pk)); // mass-weighted fall speed (with density factor) - V_nit(pk).set(qi_gt_small, p3_ice_sed_knob * table_val_ni_fallspd * rhofaci(pk)); // number-weighted fall speed (with density factor) + V_qit(pk).set(qi_gt_small, ice_sedimentation_factor * table_val_qi_fallspd * rhofaci(pk)); // mass-weighted fall speed (with density factor) + V_nit(pk).set(qi_gt_small, ice_sedimentation_factor * table_val_ni_fallspd * rhofaci(pk)); // number-weighted fall speed (with density factor) } const auto Co_max_local = max(qi_gt_small, 0, V_qit(pk) * dt_left * inv_dz(pk)); diff --git a/components/eamxx/src/physics/p3/impl/p3_init_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_init_impl.hpp new file mode 100644 index 000000000000..dff58ba33010 --- /dev/null +++ b/components/eamxx/src/physics/p3/impl/p3_init_impl.hpp @@ -0,0 +1,349 @@ +#ifndef P3_INIT_IMPL_HPP +#define P3_INIT_IMPL_HPP + +#include "p3_functions.hpp" // for ETI only but harmless for GPU + +#include "ekat/util/ekat_file_utils.hpp" + +#include + +namespace scream { +namespace p3 { + +namespace { + +template +void read_ice_lookup_tables(const bool masterproc, const char* p3_lookup_base, const char* p3_version, IceT& ice_table_vals, CollT& collect_table_vals, int densize, int rimsize, int isize, int rcollsize) +{ + using DeviceIcetable = typename IceT::non_const_type; + using DeviceColtable = typename CollT::non_const_type; + + const auto ice_table_vals_d = DeviceIcetable("ice_table_vals"); + const auto collect_table_vals_d = DeviceColtable("collect_table_vals"); + + const auto ice_table_vals_h = Kokkos::create_mirror_view(ice_table_vals_d); + const auto collect_table_vals_h = Kokkos::create_mirror_view(collect_table_vals_d); + + // + // read in ice microphysics table into host views. We always read these as doubles. + // + + std::string filename = std::string(p3_lookup_base) + std::string(p3_version); + + if (masterproc) { + std::cout << "Reading ice lookup tables in file: " << filename << std::endl; + } + + std::ifstream in(filename); + + // read header + std::string version, version_val; + in >> version >> version_val; + EKAT_REQUIRE_MSG(version == "VERSION", "Bad " << filename << ", expected VERSION X.Y.Z header"); + EKAT_REQUIRE_MSG(version_val == p3_version, "Bad " << filename << ", expected version " << p3_version << ", but got " << version_val); + + // read tables + double dum_s; int dum_i; // dum_s needs to be double to stream correctly + for (int jj = 0; jj < densize; ++jj) { + for (int ii = 0; ii < rimsize; ++ii) { + for (int i = 0; i < isize; ++i) { + in >> dum_i >> dum_i; + int j_idx = 0; + for (int j = 0; j < 15; ++j) { + in >> dum_s; + if (j > 1 && j != 10) { + ice_table_vals_h(jj, ii, i, j_idx++) = dum_s; + } + } + } + + for (int i = 0; i < isize; ++i) { + for (int j = 0; j < rcollsize; ++j) { + in >> dum_i >> dum_i; + int k_idx = 0; + for (int k = 0; k < 6; ++k) { + in >> dum_s; + if (k == 3 || k == 4) { + collect_table_vals_h(jj, ii, i, j, k_idx++) = std::log10(dum_s); + } + } + } + } + } + } + + // deep copy to device + Kokkos::deep_copy(ice_table_vals_d, ice_table_vals_h); + Kokkos::deep_copy(collect_table_vals_d, collect_table_vals_h); + ice_table_vals = ice_table_vals_d; + collect_table_vals = collect_table_vals_d; +} + +template +void compute_tables(const bool masterproc, MuRT& mu_r_table_vals, VNT& vn_table_vals, VMT& vm_table_vals, RevapT& revap_table_vals) +{ + using c = scream::physics::Constants; + + int ii,jj,kk; + S lamr,mu_r,dm,dum1,dum2,dum3,dum4,dum5,dd,amg,vt,dia; + + using MuRT_NC = typename MuRT::non_const_type; + using VNT_NC = typename VNT::non_const_type; + using VMT_NC = typename VMT::non_const_type; + using RevapT_NC = typename RevapT::non_const_type; + + MuRT_NC mu_r_table_vals_nc("mu_r_table_vals"); + VNT_NC vn_table_vals_nc("vn_table_vals"); + VMT_NC vm_table_vals_nc("vm_table_vals"); + RevapT_NC revap_table_vals_nc("revap_table_vals"); + + // Get host views + auto mu_r_table_vals_h = Kokkos::create_mirror_view(mu_r_table_vals_nc); + auto revap_table_vals_h = Kokkos::create_mirror_view(revap_table_vals_nc); + auto vn_table_vals_h = Kokkos::create_mirror_view(vn_table_vals_nc); + auto vm_table_vals_h = Kokkos::create_mirror_view(vm_table_vals_nc); + + if (masterproc) { + std::cout << "Recomputing lookup (non-ice) tables" << std::endl; + } + + // ------------------------------------------------------------------------------------------ + + // Generate lookup table for rain shape parameter mu_r + // this is very fast so it can be generated at the start of each run + // make a 150x1 1D lookup table, this is done in parameter + // space of a scaled mean size proportional qr/Nr -- initlamr + + // write(iulog,*) ' Generating rain lookup-table ...' + + // AaronDonahue: Switching to table ver 4 means switching to a constand mu_r, + // so this section is commented out. + Kokkos::deep_copy(mu_r_table_vals_h, 1); // mu_r_constant =1. In other places, this is runtime_options.constant_mu_rain + + static constexpr S thrd = 1./3; + static constexpr S small = 1.e-30; + + //....................................................................... + // Generate lookup table for rain fallspeed and ventilation parameters + // the lookup table is two dimensional as a function of number-weighted mean size + // proportional to qr/Nr and shape parameter mu_r + for (ii = 1; ii <= 10; ++ii) { + mu_r = 1; // mu_r_constant = 1 + + // loop over number-weighted mean size + for (jj = 1; jj <= 300; ++jj) { + if (jj <= 20) { + dm = (jj*10 - 5)*1.e-6; // mean size [m] + } + else { + dm = ((jj-20)*30 + 195)*1.e-6; // mean size [m] + } + + lamr = (mu_r + 1)/dm; + + // do numerical integration over PSD + + dum1 = 0; // numerator, number-weighted fallspeed + dum2 = 0; // denominator, number-weighted fallspeed + dum3 = 0; // numerator, mass-weighted fallspeed + dum4 = 0; // denominator, mass-weighted fallspeed + dum5 = 0; // term for ventilation factor in evap + dd = 2; + + // loop over PSD to numerically integrate number and mass-weighted mean fallspeeds + for (kk = 1; kk <= 10000; ++kk) { + + dia = (kk*dd - dd*0.5)*1.e-6; // size bin [m] + amg = c::PIOV6*997 * std::pow(dia, 3); // mass [kg] + amg = amg*1000; // convert [kg] to [g] + + // get fallspeed as a function of size [m s-1] + if (dia*1.e+6 <= 134.43) { + vt = 4.5795e+3 * std::pow(amg, 2*thrd); + } + else if (dia*1.e+6 < 1511.64) { + vt = 4.962e+1 * std::pow(amg, thrd); + } + else if (dia*1.e+6 < 3477.84) { + vt = 1.732e+1 * std::pow(amg, c::SXTH); + } + else { + vt = 9.17; + } + + // note: factor of 4.*mu_r is non-answer changing and only needed to + // prevent underflow/overflow errors, same with 3.*mu_r for dum5 + dum1 += vt * std::pow(10, mu_r*std::log10(dia) + 4*mu_r) * std::exp(-lamr*dia) * dd * 1.e-6; + dum2 += std::pow(10, mu_r*std::log10(dia) + 4*mu_r) * std::exp(-lamr*dia) * dd * 1.e-6; + dum3 += vt * std::pow(10, (mu_r+3)*std::log10(dia) + 4*mu_r) * std::exp(-lamr*dia) * dd * 1.e-6; + dum4 += std::pow(10, (mu_r+3)*std::log10(dia) + 4*mu_r) * std::exp(-lamr*dia) * dd * 1.e-6; + dum5 += std::pow(vt*dia, 0.5) * std::pow(10, (mu_r+1)*std::log10(dia) + 3*mu_r) * std::exp(-lamr*dia) * dd * 1.e-6; + } + + dum2 = std::max(dum2, small); // to prevent divide-by-zero below + dum4 = std::max(dum4, small); // to prevent divide-by-zero below + dum5 = std::max(dum5, small); // to prevent log10-of-zero below + + vn_table_vals_h(jj-1,ii-1) = dum1/dum2; + vm_table_vals_h(jj-1,ii-1) = dum3/dum4; + revap_table_vals_h(jj-1,ii-1) = std::pow(10, std::log10(dum5) + (mu_r+1)*std::log10(lamr) - (3*mu_r)); + } + } + + Kokkos::deep_copy(mu_r_table_vals_nc, mu_r_table_vals_h); + Kokkos::deep_copy(revap_table_vals_nc, revap_table_vals_h); + Kokkos::deep_copy(vn_table_vals_nc, vn_table_vals_h); + Kokkos::deep_copy(vm_table_vals_nc, vm_table_vals_h); + + mu_r_table_vals = mu_r_table_vals_nc; + vn_table_vals = vn_table_vals_nc; + vm_table_vals = vm_table_vals_nc; + revap_table_vals = revap_table_vals_nc; +} + +template +static void action(const ekat::FILEPtr& fid, S* data, const size_t size) +{ + if constexpr (IsRead) { + ekat::read(data, size, fid); + } + else { + ekat::write(data, size, fid); + } +} + +template +void io_impl(const bool masterproc, const char* dir, MuRT& mu_r_table_vals, VNT& vn_table_vals, VMT& vm_table_vals, RevapT& revap_table_vals) +{ + if (masterproc) { + std::cout << (IsRead ? "Reading" : "Writing") << " lookup (non-ice) tables in dir " << dir << std::endl; + } + + std::string extension = +#ifdef SCREAM_DOUBLE_PRECISION + "8" +#else + "4" +#endif + ; + + const char* rw_flag = IsRead ? "r" : "w"; + + // Get host views + auto mu_r_table_vals_h = Kokkos::create_mirror_view(mu_r_table_vals); + auto revap_table_vals_h = Kokkos::create_mirror_view(revap_table_vals); + auto vn_table_vals_h = Kokkos::create_mirror_view(vn_table_vals); + auto vm_table_vals_h = Kokkos::create_mirror_view(vm_table_vals); + + // Add v2 because these tables are not identical to v1 due to roundoff differences + // caused by doing the math in C++ instead of f90. + std::string mu_r_filename = std::string(dir) + "/mu_r_table_vals_v2.dat" + extension; + std::string revap_filename = std::string(dir) + "/revap_table_vals_v2.dat" + extension; + std::string vn_filename = std::string(dir) + "/vn_table_vals_v2.dat" + extension; + std::string vm_filename = std::string(dir) + "/vm_table_vals_v2.dat" + extension; + + ekat::FILEPtr mu_r_file(fopen(mu_r_filename.c_str(), rw_flag)); + ekat::FILEPtr revap_file(fopen(revap_filename.c_str(), rw_flag)); + ekat::FILEPtr vn_file(fopen(vn_filename.c_str(), rw_flag)); + ekat::FILEPtr vm_file(fopen(vm_filename.c_str(), rw_flag)); + + // Read files + action(mu_r_file, mu_r_table_vals_h.data(), mu_r_table_vals.size()); + action(revap_file, revap_table_vals_h.data(), revap_table_vals.size()); + action(vn_file, vn_table_vals_h.data(), vn_table_vals.size()); + action(vm_file, vm_table_vals_h.data(), vm_table_vals.size()); + + // Copy back to device + if constexpr (IsRead) { + Kokkos::deep_copy(mu_r_table_vals, mu_r_table_vals_h); + Kokkos::deep_copy(revap_table_vals, revap_table_vals_h); + Kokkos::deep_copy(vn_table_vals, vn_table_vals_h); + Kokkos::deep_copy(vm_table_vals, vm_table_vals_h); + } +} + +template +void read_computed_tables(const bool masterproc, const char* dir, MuRT& mu_r_table_vals, VNT& vn_table_vals, VMT& vm_table_vals, RevapT& revap_table_vals) +{ + using MuRT_NC = typename MuRT::non_const_type; + using VNT_NC = typename VNT::non_const_type; + using VMT_NC = typename VMT::non_const_type; + using RevapT_NC = typename RevapT::non_const_type; + + MuRT_NC mu_r_table_vals_nc("mu_r_table_vals"); + VNT_NC vn_table_vals_nc("vn_table_vals"); + VMT_NC vm_table_vals_nc("vm_table_vals"); + RevapT_NC revap_table_vals_nc("revap_table_vals"); + + io_impl(masterproc, dir, mu_r_table_vals_nc, vn_table_vals_nc, vm_table_vals_nc, revap_table_vals_nc); + + mu_r_table_vals = mu_r_table_vals_nc; + vn_table_vals = vn_table_vals_nc; + vm_table_vals = vm_table_vals_nc; + revap_table_vals = revap_table_vals_nc; +} + +template +void write_computed_tables(const bool masterproc, const char* dir, const MuRT& mu_r_table_vals, const VNT& vn_table_vals, const VMT& vm_table_vals, const RevapT& revap_table_vals) +{ + io_impl(masterproc, dir, mu_r_table_vals, vn_table_vals, vm_table_vals, revap_table_vals); +} + +template +void compute_dnu(DnuT& dnu_table_vals) +{ + typename DnuT::non_const_type dnu_table_vals_non_const("dnu_table_vals"); + const auto dnu_table_h = Kokkos::create_mirror_view(dnu_table_vals_non_const); + dnu_table_h(0) = 0.000; + dnu_table_h(1) = -0.557; + dnu_table_h(2) = -0.430; + dnu_table_h(3) = -0.307; + dnu_table_h(4) = -0.186; + dnu_table_h(5) = -0.067; + dnu_table_h(6) = -0.050; + dnu_table_h(7) = -0.167; + dnu_table_h(8) = -0.282; + dnu_table_h(9) = -0.397; + dnu_table_h(10) = -0.512; + dnu_table_h(11) = -0.626; + dnu_table_h(12) = -0.739; + dnu_table_h(13) = -0.853; + dnu_table_h(14) = -0.966; + dnu_table_h(15) = -0.966; + Kokkos::deep_copy(dnu_table_vals_non_const, dnu_table_h); + dnu_table_vals = DnuT(dnu_table_vals_non_const); +} + +} + +/* + * Implementation of p3 init. Clients should NOT #include + * this file, #include p3_functions.hpp instead. + */ +template +typename Functions::P3LookupTables Functions +::p3_init (const bool write_tables, const bool masterproc) { + P3LookupTables lookup_tables; // This struct could be our global singleton + auto version = P3C::p3_version; + auto p3_lookup_base = P3C::p3_lookup_base; + static const char* dir = SCREAM_DATA_DIR "/tables"; + // p3_init_a (reads ice_table, collect_table) + read_ice_lookup_tables(masterproc, p3_lookup_base, version, lookup_tables.ice_table_vals, lookup_tables.collect_table_vals, P3C::densize, P3C::rimsize, P3C::isize, P3C::rcollsize); + if (write_tables) { + //p3_init_b (computes tables mu_r_table, revap_table, vn_table, vm_table) + compute_tables(masterproc, lookup_tables.mu_r_table_vals, lookup_tables.vn_table_vals, lookup_tables.vm_table_vals, lookup_tables.revap_table_vals); + write_computed_tables(masterproc, dir, lookup_tables.mu_r_table_vals, lookup_tables.vn_table_vals, lookup_tables.vm_table_vals, lookup_tables.revap_table_vals); + } + else { + read_computed_tables(masterproc, dir, lookup_tables.mu_r_table_vals, lookup_tables.vn_table_vals, lookup_tables.vm_table_vals, lookup_tables.revap_table_vals); + } + // dnu is always computed/hardcoded + compute_dnu(lookup_tables.dnu_table_vals); + + return lookup_tables; +} + +} // namespace p3 +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp index 964ec21be700..fb76ac0290e8 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp @@ -83,8 +83,7 @@ ::p3_main_internal( const P3LookupTables& lookup_tables, const WorkspaceManager& workspace_mgr, Int nj, - Int nk, - const physics::P3_Constants & p3constants) + Int nk) { using ExeSpace = typename KT::ExeSpace; using ScratchViewType = Kokkos::View; @@ -100,6 +99,8 @@ ::p3_main_internal( const Int kbot = kdir == -1 ? nk-1 : 0; constexpr bool debug_ABORT = false; + const bool do_ice_production = runtime_options.do_ice_production; + // we do not want to measure init stuff auto start = std::chrono::steady_clock::now(); @@ -230,7 +231,7 @@ ::p3_main_internal( T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, oqv, oth, oqc, onc, oqr, onr, oqi, oni, oqm, obm, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, - ni_incld, bm_incld, nucleationPossible, hydrometeorsPresent, p3constants); + ni_incld, bm_incld, nucleationPossible, hydrometeorsPresent, runtime_options); // There might not be any work to do for this team if (!(nucleationPossible || hydrometeorsPresent)) { @@ -250,7 +251,7 @@ ::p3_main_internal( nr_incld, ni_incld, bm_incld, mu_c, nu, lamc, cdist, cdist1, cdistr, mu_r, lamr, logn0r, oqv2qi_depos_tend, oprecip_total_tend, onevapr, qr_evap_tend, ovap_liq_exchange, ovap_ice_exchange, oliq_ice_exchange, - pratot, prctot, hydrometeorsPresent, nk, p3constants); + pratot, prctot, hydrometeorsPresent, nk, runtime_options); //NOTE: At this point, it is possible to have negative (but small) nc, nr, ni. This is not // a problem; those values get clipped to zero in the sedimentation section (if necessary). @@ -278,19 +279,20 @@ ::p3_main_internal( rho, inv_rho, rhofacr, ocld_frac_r, inv_dz, qr_incld, team, workspace, lookup_tables.vn_table_vals, lookup_tables.vm_table_vals, nk, ktop, kbot, kdir, infrastructure.dt, inv_dt, oqr, onr, nr_incld, mu_r, lamr, oprecip_liq_flux, qtend_ignore, ntend_ignore, - diagnostic_outputs.precip_liq_surf(i), p3constants); + diagnostic_outputs.precip_liq_surf(i), runtime_options); // Ice sedimentation: (adaptive substepping) ice_sedimentation( rho, inv_rho, rhofaci, ocld_frac_i, inv_dz, team, workspace, nk, ktop, kbot, kdir, infrastructure.dt, inv_dt, oqi, qi_incld, oni, ni_incld, oqm, qm_incld, obm, bm_incld, qtend_ignore, ntend_ignore, - lookup_tables.ice_table_vals, diagnostic_outputs.precip_ice_surf(i), p3constants); + lookup_tables.ice_table_vals, diagnostic_outputs.precip_ice_surf(i), runtime_options); // homogeneous freezing of cloud and rain - homogeneous_freezing( - T_atm, oinv_exner, team, nk, ktop, kbot, kdir, oqc, onc, oqr, onr, oqi, - oni, oqm, obm, oth); + if(do_ice_production) { + homogeneous_freezing(T_atm, oinv_exner, team, nk, ktop, kbot, kdir, oqc, + onc, oqr, onr, oqi, oni, oqm, obm, oth); + } // // final checks to ensure consistency of mass/number @@ -301,7 +303,7 @@ ::p3_main_internal( rho, inv_rho, rhofaci, oqv, oth, oqc, onc, oqr, onr, oqi, oni, oqm, obm, mu_c, nu, lamc, mu_r, lamr, ovap_liq_exchange, ze_rain, ze_ice, diag_vm_qi, odiag_eff_radius_qi, diag_diam_qi, - orho_qi, diag_equiv_reflectivity, odiag_eff_radius_qc, odiag_eff_radius_qr, p3constants); + orho_qi, diag_equiv_reflectivity, odiag_eff_radius_qc, odiag_eff_radius_qr, runtime_options); // // merge ice categories with similar properties @@ -343,8 +345,7 @@ ::p3_main( #endif const WorkspaceManager& workspace_mgr, Int nj, - Int nk, - const physics::P3_Constants & p3constants) + Int nk) { #ifdef SCREAM_P3_SMALL_KERNELS return p3_main_internal_disp(runtime_options, @@ -356,7 +357,7 @@ ::p3_main( lookup_tables, temporaries, workspace_mgr, - nj, nk, p3constants); + nj, nk); #else return p3_main_internal(runtime_options, prognostic_state, @@ -366,7 +367,7 @@ ::p3_main( history_only, lookup_tables, workspace_mgr, - nj, nk, p3constants); + nj, nk); #endif } } // namespace p3 diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl_part1.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl_part1.hpp index 6771a48d4c68..e6f362c1d813 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl_part1.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl_part1.hpp @@ -63,7 +63,7 @@ ::p3_main_part1( const uview_1d& bm_incld, bool& nucleationPossible, bool& hydrometeorsPresent, - const physics::P3_Constants & p3constants) + const P3Runtime& runtime_options) { // Get access to saturation functions using physics = scream::physics::Functions; @@ -80,7 +80,7 @@ ::p3_main_part1( constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; - const Scalar p3_spa_to_nc = p3constants.p3_spa_to_nc; + const Scalar spa_ccn_to_nc_factor = runtime_options.spa_ccn_to_nc_factor; nucleationPossible = false; hydrometeorsPresent = false; @@ -132,15 +132,16 @@ ::p3_main_part1( // adjustment already applied in macrophysics. If prescribed drop number is used, this is also a good place to // prescribe that value - if (do_prescribed_CCN) { - nc(k).set(not_drymass, max(nc(k), p3_spa_to_nc*nccn_prescribed(k)/inv_cld_frac_l(k))); - } else if (predictNc) { - nc(k).set(not_drymass, max(nc(k) + nc_nuceat_tend(k) * dt, 0.0)); + if(do_prescribed_CCN) { + nc(k).set(not_drymass, + max(nc(k), spa_ccn_to_nc_factor * nccn_prescribed(k) / + inv_cld_frac_l(k))); + } else if(predictNc) { + nc(k).set(not_drymass, max(nc(k) + nc_nuceat_tend(k) * dt, 0.0)); } else { - // nccnst is in units of #/m3 so needs to be converted. - nc(k).set(not_drymass, nccnst*inv_rho(k)); + // nccnst is in units of #/m3 so needs to be converted. + nc(k).set(not_drymass, nccnst * inv_rho(k)); } - } drymass = qr(k) < qsmall; diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl_part2.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl_part2.hpp index d19dc579b7e9..3214549a3b65 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl_part2.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl_part2.hpp @@ -92,7 +92,7 @@ ::p3_main_part2( const uview_1d& pratot, const uview_1d& prctot, bool& hydrometeorsPresent, const Int& nk, - const physics::P3_Constants & p3constants) + const P3Runtime& runtime_options) { constexpr Scalar qsmall = C::QSMALL; constexpr Scalar nsmall = C::NSMALL; @@ -104,6 +104,8 @@ ::p3_main_part2( constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; + const bool do_ice_production = runtime_options.do_ice_production; + team.team_barrier(); hydrometeorsPresent = false; team.team_barrier(); @@ -215,7 +217,7 @@ ::p3_main_part2( lamc(k), cdist(k), cdist1(k), not_skip_micro); nc(k).set(not_skip_micro, nc_incld(k) * cld_frac_l(k)); - get_rain_dsd2(qr_incld(k), nr_incld(k), mu_r(k), lamr(k), p3constants, not_skip_micro); + get_rain_dsd2(qr_incld(k), nr_incld(k), mu_r(k), lamr(k), runtime_options, not_skip_micro); get_cdistr_logn0r(qr_incld(k), nr_incld(k), mu_r(k), lamr(k), cdistr(k), logn0r(k), not_skip_micro); nr(k).set(not_skip_micro, nr_incld(k) * cld_frac_r(k)); @@ -228,7 +230,7 @@ ::p3_main_part2( ni_incld(k).set(qi_gt_small, max(ni_incld(k), nsmall)); nr_incld(k).set(qi_gt_small, max(nr_incld(k), nsmall)); - const auto rhop = calc_bulk_rho_rime(qi_incld(k), qm_incld(k), bm_incld(k), p3constants, qi_gt_small); + const auto rhop = calc_bulk_rho_rime(qi_incld(k), qm_incld(k), bm_incld(k), runtime_options, qi_gt_small); qm(k).set(qi_gt_small, qm_incld(k)*cld_frac_i(k) ); bm(k).set(qi_gt_small, bm_incld(k)*cld_frac_i(k) ); @@ -266,61 +268,77 @@ ::p3_main_part2( // ice processes // ...................................................................... - // collection of droplets - ice_cldliq_collection( - rho(k), T_atm(k), rhofaci(k), table_val_qc2qi_collect, qi_incld(k), qc_incld(k), ni_incld(k), nc_incld(k), - qc2qi_collect_tend, nc_collect_tend, qc2qr_ice_shed_tend, ncshdc, p3constants, not_skip_micro); - - // collection of rain - ice_rain_collection( - rho(k), T_atm(k), rhofaci(k), logn0r(k), table_val_nr_collect, table_val_qr2qi_collect, qi_incld(k), ni_incld(k), qr_incld(k), - qr2qi_collect_tend, nr_collect_tend, p3constants, not_skip_micro); - - // collection between ice categories - - // PMC nCat deleted lots of stuff here. - - // self-collection of ice - ice_self_collection( - rho(k), rhofaci(k), table_val_ni_self_collect, eii, qm_incld(k), qi_incld(k), ni_incld(k), - ni_selfcollect_tend, not_skip_micro); + if(do_ice_production) { + // collection of droplets + ice_cldliq_collection(rho(k), T_atm(k), rhofaci(k), + table_val_qc2qi_collect, qi_incld(k), qc_incld(k), + ni_incld(k), nc_incld(k), qc2qi_collect_tend, + nc_collect_tend, qc2qr_ice_shed_tend, ncshdc, + runtime_options, not_skip_micro); + + // collection of rain + ice_rain_collection(rho(k), T_atm(k), rhofaci(k), logn0r(k), + table_val_nr_collect, table_val_qr2qi_collect, + qi_incld(k), ni_incld(k), qr_incld(k), + qr2qi_collect_tend, nr_collect_tend, + runtime_options, not_skip_micro); + + // collection between ice categories + + // PMC nCat deleted lots of stuff here. + + // self-collection of ice + ice_self_collection(rho(k), rhofaci(k), table_val_ni_self_collect, eii, + qm_incld(k), qi_incld(k), ni_incld(k), + ni_selfcollect_tend, not_skip_micro); + } // melting - ice_melting( - rho(k), T_atm(k), pres(k), rhofaci(k), table_val_qi2qr_melting, table_val_qi2qr_vent_melt, dv, sc, mu, kap, qv(k), qi_incld(k), ni_incld(k), - qi2qr_melt_tend, ni2nr_melt_tend, not_skip_micro); - - // calculate wet growth - ice_cldliq_wet_growth( - rho(k), T_atm(k), pres(k), rhofaci(k), table_val_qi2qr_melting, table_val_qi2qr_vent_melt, - dv, kap, mu, sc, qv(k), qc_incld(k), qi_incld(k), ni_incld(k), qr_incld(k), - wetgrowth, qr2qi_collect_tend, qc2qi_collect_tend, qc_growth_rate, nr_ice_shed_tend, qc2qr_ice_shed_tend, not_skip_micro); - - // calculate total inverse ice relaxation timescale combined for all ice categories - // note 'f1pr' values are normalized, so we need to multiply by N + ice_melting(rho(k), T_atm(k), pres(k), rhofaci(k), + table_val_qi2qr_melting, table_val_qi2qr_vent_melt, dv, sc, + mu, kap, qv(k), qi_incld(k), ni_incld(k), qi2qr_melt_tend, + ni2nr_melt_tend, not_skip_micro); + + if(do_ice_production) { + // calculate wet growth + ice_cldliq_wet_growth( + rho(k), T_atm(k), pres(k), rhofaci(k), table_val_qi2qr_melting, + table_val_qi2qr_vent_melt, dv, kap, mu, sc, qv(k), qc_incld(k), + qi_incld(k), ni_incld(k), qr_incld(k), wetgrowth, + qr2qi_collect_tend, qc2qi_collect_tend, qc_growth_rate, + nr_ice_shed_tend, qc2qr_ice_shed_tend, not_skip_micro); + } + + // calculate total inverse ice relaxation timescale combined for all ice + // categories note 'f1pr' values are normalized, so we need to multiply + // by N ice_relaxation_timescale( - rho(k), T_atm(k), rhofaci(k), table_val_qi2qr_melting, table_val_qi2qr_vent_melt, dv, mu, sc, qi_incld(k), ni_incld(k), - epsi, epsi_tot, not_skip_micro); + rho(k), T_atm(k), rhofaci(k), table_val_qi2qr_melting, + table_val_qi2qr_vent_melt, dv, mu, sc, qi_incld(k), ni_incld(k), epsi, + epsi_tot, not_skip_micro); // calculate rime density - calc_rime_density( - T_atm(k), rhofaci(k), table_val_qi_fallspd, acn(k), lamc(k), mu_c(k), qc_incld(k), qc2qi_collect_tend, - vtrmi1, rho_qm_cloud, not_skip_micro); - - // contact and immersion freezing droplets - cldliq_immersion_freezing( - T_atm(k), lamc(k), mu_c(k), cdist1(k), qc_incld(k), inv_qc_relvar(k), - qc2qi_hetero_freeze_tend, nc2ni_immers_freeze_tend, p3constants, not_skip_micro); - - // for future: get rid of log statements below for rain freezing - rain_immersion_freezing( - T_atm(k), lamr(k), mu_r(k), cdistr(k), qr_incld(k), - qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend, p3constants, not_skip_micro); - - // rime splintering (Hallet-Mossop 1974) - // PMC comment: Morrison and Milbrandt 2015 part 1 and 2016 part 3 both say - // that Hallet-Mossop should be neglected if 1 category to compensate for - // artificial smearing out of ice DSD + calc_rime_density(T_atm(k), rhofaci(k), table_val_qi_fallspd, acn(k), + lamc(k), mu_c(k), qc_incld(k), qc2qi_collect_tend, + vtrmi1, rho_qm_cloud, not_skip_micro); + + if(do_ice_production) { + // contact and immersion freezing droplets + cldliq_immersion_freezing( + T_atm(k), lamc(k), mu_c(k), cdist1(k), qc_incld(k), + inv_qc_relvar(k), qc2qi_hetero_freeze_tend, + nc2ni_immers_freeze_tend, runtime_options, not_skip_micro); + + // for future: get rid of log statements below for rain freezing + rain_immersion_freezing(T_atm(k), lamr(k), mu_r(k), cdistr(k), + qr_incld(k), qr2qi_immers_freeze_tend, + nr2ni_immers_freeze_tend, runtime_options, + not_skip_micro); + // rime splintering (Hallet-Mossop 1974) + // PMC comment: Morrison and Milbrandt 2015 part 1 and 2016 part 3 both + // say that Hallet-Mossop should be neglected if 1 category to + // compensate for artificial smearing out of ice DSD + } // ................................................ // condensation/evaporation/deposition/sublimation @@ -336,21 +354,28 @@ ::p3_main_part2( ab,abi,epsr,epsi_tot,T_atm(k),t_prev(k),dqsdt,dt, qr2qv_evap_tend,nr_evap_tend, not_skip_micro); - ice_deposition_sublimation( - qi_incld(k), ni_incld(k), T_atm(k), qv_sat_l(k), qv_sat_i(k), epsi, abi, qv(k), inv_dt, - qv2qi_vapdep_tend, qi2qv_sublim_tend, ni_sublim_tend, qc2qi_berg_tend, not_skip_micro); + if(do_ice_production) { + ice_deposition_sublimation( + qi_incld(k), ni_incld(k), T_atm(k), qv_sat_l(k), qv_sat_i(k), epsi, + abi, qv(k), inv_dt, qv2qi_vapdep_tend, qi2qv_sublim_tend, + ni_sublim_tend, qc2qi_berg_tend, not_skip_micro); + } + } // deposition/condensation-freezing nucleation - ice_nucleation( - T_atm(k), inv_rho(k), ni(k), ni_activated(k), qv_supersat_i(k), inv_dt, predictNc, do_prescribed_CCN, - qv2qi_nucleat_tend, ni_nucleat_tend, p3constants, not_skip_all); + if(do_ice_production) { + ice_nucleation(T_atm(k), inv_rho(k), ni(k), ni_activated(k), + qv_supersat_i(k), inv_dt, predictNc, do_prescribed_CCN, + qv2qi_nucleat_tend, ni_nucleat_tend, runtime_options, + not_skip_all); + } // cloud water autoconversion // NOTE: cloud_water_autoconversion must be called before droplet_self_collection cloud_water_autoconversion( rho(k), qc_incld(k), nc_incld(k), inv_qc_relvar(k), - qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr, p3constants, not_skip_all); + qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr, runtime_options, not_skip_all); // self-collection of droplets droplet_self_collection( @@ -360,13 +385,13 @@ ::p3_main_part2( // accretion of cloud by rain cloud_rain_accretion( rho(k), inv_rho(k), qc_incld(k), nc_incld(k), qr_incld(k), inv_qc_relvar(k), - qc2qr_accret_tend, nc_accret_tend, p3constants, not_skip_all); + qc2qr_accret_tend, nc_accret_tend, runtime_options, not_skip_all); // self-collection and breakup of rain // (breakup following modified Verlinde and Cotton scheme) rain_self_collection( rho(k), qr_incld(k), nr_incld(k), - nr_selfcollect_tend, p3constants, not_skip_all); + nr_selfcollect_tend, runtime_options, not_skip_all); // Here we map the microphysics tendency rates back to CELL-AVERAGE quantities for updating // cell-average quantities. diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl_part3.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl_part3.hpp index febb00ef4d31..72bc7e8f577e 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl_part3.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl_part3.hpp @@ -56,7 +56,7 @@ ::p3_main_part3( const uview_1d& diag_equiv_reflectivity, const uview_1d& diag_eff_radius_qc, const uview_1d& diag_eff_radius_qr, - const physics::P3_Constants & p3constants) + const P3Runtime& runtime_options) { constexpr Scalar qsmall = C::QSMALL; constexpr Scalar inv_cp = C::INV_CP; @@ -108,7 +108,7 @@ ::p3_main_part3( auto nr_incld = nr(k)/cld_frac_r(k); //nr_incld is updated in get_rain_dsd2 but isn't used again get_rain_dsd2( - qr_incld, nr_incld, mu_r(k), lamr(k), p3constants, qr_gt_small); + qr_incld, nr_incld, mu_r(k), lamr(k), runtime_options, qr_gt_small); //Note that integrating over the drop-size PDF as done here should only be done to in-cloud //quantities but radar reflectivity is likely meant to be a cell ave. Thus nr in the next line @@ -143,7 +143,7 @@ ::p3_main_part3( auto qm_incld = qm(k)/cld_frac_i(k); auto bm_incld = bm(k)/cld_frac_i(k); - const auto rhop = calc_bulk_rho_rime(qi_incld, qm_incld, bm_incld, p3constants, qi_gt_small); + const auto rhop = calc_bulk_rho_rime(qi_incld, qm_incld, bm_incld, runtime_options, qi_gt_small); qm(k).set(qi_gt_small, qm_incld*cld_frac_i(k) ); bm(k).set(qi_gt_small, bm_incld*cld_frac_i(k) ); diff --git a/components/eamxx/src/physics/p3/impl/p3_rain_imm_freezing_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_rain_imm_freezing_impl.hpp index c960698e4b18..f42daad861c8 100644 --- a/components/eamxx/src/physics/p3/impl/p3_rain_imm_freezing_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_rain_imm_freezing_impl.hpp @@ -17,7 +17,7 @@ void Functions ::rain_immersion_freezing(const Spack& T_atm, const Spack& lamr, const Spack& mu_r, const Spack& cdistr, const Spack& qr_incld, Spack& qr2qi_immers_freeze_tend, Spack& nr2ni_immers_freeze_tend, - const physics::P3_Constants & p3constants, + const P3Runtime& runtime_options, const Smask& context) { constexpr Scalar qsmall = C::QSMALL; @@ -26,19 +26,22 @@ ::rain_immersion_freezing(const Spack& T_atm, const Spack& lamr, constexpr Scalar CONS5 = C::CONS5; constexpr Scalar CONS6 = C::CONS6; - const Scalar p3_a_imm = p3constants.p3_a_imm; + const Scalar immersion_freezing_exponent = + runtime_options.immersion_freezing_exponent; const auto qr_not_small_and_t_freezing = (qr_incld >= qsmall) && (T_atm <= T_rainfrz) && context; - if (qr_not_small_and_t_freezing.any()) { - qr2qi_immers_freeze_tend.set(qr_not_small_and_t_freezing, - CONS6 * - exp(log(cdistr) + log(tgamma(sp(7.)+mu_r)) - sp(6.)*log(lamr)) * - exp(p3_a_imm*(T_zerodegc-T_atm))); - nr2ni_immers_freeze_tend.set(qr_not_small_and_t_freezing, - CONS5 * - exp(log(cdistr) + log(tgamma(sp(4.)+mu_r)) - sp(3.)*log(lamr)) * - exp(p3_a_imm*(T_zerodegc-T_atm))); + if(qr_not_small_and_t_freezing.any()) { + qr2qi_immers_freeze_tend.set( + qr_not_small_and_t_freezing, + CONS6 * + exp(log(cdistr) + log(tgamma(sp(7.) + mu_r)) - sp(6.) * log(lamr)) * + exp(immersion_freezing_exponent * (T_zerodegc - T_atm))); + nr2ni_immers_freeze_tend.set( + qr_not_small_and_t_freezing, + CONS5 * + exp(log(cdistr) + log(tgamma(sp(4.) + mu_r)) - sp(3.) * log(lamr)) * + exp(immersion_freezing_exponent * (T_zerodegc - T_atm))); } } diff --git a/components/eamxx/src/physics/p3/impl/p3_rain_sed_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_rain_sed_impl.hpp index 558264305f17..d35fe1a3f1d4 100644 --- a/components/eamxx/src/physics/p3/impl/p3_rain_sed_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_rain_sed_impl.hpp @@ -18,11 +18,11 @@ ::compute_rain_fall_velocity( const view_2d_table& vn_table_vals, const view_2d_table& vm_table_vals, const Spack& qr_incld, const Spack& rhofacr, Spack& nr_incld, Spack& mu_r, Spack& lamr, Spack& V_qr, Spack& V_nr, - const physics::P3_Constants & p3constants, + const P3Runtime& runtime_options, const Smask& context) { Table3 table; - get_rain_dsd2(qr_incld, nr_incld, mu_r, lamr, p3constants, context); + get_rain_dsd2(qr_incld, nr_incld, mu_r, lamr, runtime_options, context); if (context.any()) { lookup(mu_r, lamr, table, context); @@ -56,7 +56,7 @@ ::rain_sedimentation( const uview_1d& qr_tend, const uview_1d& nr_tend, Scalar& precip_liq_surf, - const physics::P3_Constants & p3constants) + const P3Runtime& runtime_options) { // Get temporary workspaces needed for the ice-sed calculation uview_1d V_qr, V_nr, flux_qx, flux_nx; @@ -112,7 +112,7 @@ ::rain_sedimentation( compute_rain_fall_velocity(vn_table_vals, vm_table_vals, qr_incld(pk), rhofacr(pk), nr_incld(pk), mu_r(pk), lamr(pk), - V_qr(pk), V_nr(pk), p3constants, qr_gt_small); + V_qr(pk), V_nr(pk), runtime_options, qr_gt_small); //in compute_rain_fall_velocity, get_rain_dsd2 keeps the drop-size //distribution within reasonable bounds by modifying nr_incld. diff --git a/components/eamxx/src/physics/p3/impl/p3_rain_self_collection_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_rain_self_collection_impl.hpp index 807e928948cb..f61aa7f6f0cc 100644 --- a/components/eamxx/src/physics/p3/impl/p3_rain_self_collection_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_rain_self_collection_impl.hpp @@ -11,7 +11,7 @@ KOKKOS_FUNCTION void Functions ::rain_self_collection( const Spack& rho, const Spack& qr_incld, const Spack& nr_incld, Spack& nr_selfcollect_tend, - const physics::P3_Constants & p3constants, + const P3Runtime& runtime_options, const Smask& context) { // ------------------------------------------------------ @@ -22,23 +22,35 @@ ::rain_self_collection( constexpr Scalar rho_h2o = C::RHO_H2O; constexpr Scalar pi = C::Pi; - const Scalar p3_d_breakup_cutoff = p3constants.p3_d_breakup_cutoff; + const Scalar rain_selfcollection_breakup_diameter = + runtime_options.rain_selfcollection_breakup_diameter; + const Scalar rain_selfcollection_prefactor = + runtime_options.rain_selfcollection_prefactor; const auto qr_incld_not_small = qr_incld >= qsmall && context; - if (qr_incld_not_small.any()) { - - const auto dum2 = cbrt((qr_incld)/(pi*rho_h2o*nr_incld)); + if(qr_incld_not_small.any()) { + // use mass-mean diameter (do this by using + // the old version of lambda w/o mu dependence) + // note there should be a factor of 6^(1/3), but we + // want to keep breakup threshold consistent so 'dum' + // is expressed in terms of lambda rather than mass-mean D + const auto dum2 = cbrt((qr_incld) / (pi * rho_h2o * nr_incld)); Spack dum; - const auto dum2_lt_dum1 = dum2 < p3_d_breakup_cutoff && qr_incld_not_small; - const auto dum2_gt_dum1 = dum2 >= p3_d_breakup_cutoff && qr_incld_not_small; + const auto dum2_lt_dum1 = + dum2 < rain_selfcollection_breakup_diameter && qr_incld_not_small; + const auto dum2_gt_dum1 = + dum2 >= rain_selfcollection_breakup_diameter && qr_incld_not_small; dum.set(dum2_lt_dum1, 1); - if (dum2_gt_dum1.any()) { - dum.set(dum2_gt_dum1, 2 - exp(2300 * (dum2-p3_d_breakup_cutoff))); + if(dum2_gt_dum1.any()) { + dum.set(dum2_gt_dum1, + 2 - exp(2300 * (dum2 - rain_selfcollection_breakup_diameter))); } - nr_selfcollect_tend.set(qr_incld_not_small, dum*sp(5.78)*nr_incld*qr_incld*rho); + nr_selfcollect_tend.set( + qr_incld_not_small, + dum * rain_selfcollection_prefactor * nr_incld * qr_incld * rho); } } diff --git a/components/eamxx/src/physics/p3/impl/p3_table3_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_table3_impl.hpp index d099cd7c731a..dbd7ea5726a4 100644 --- a/components/eamxx/src/physics/p3/impl/p3_table3_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_table3_impl.hpp @@ -84,67 +84,15 @@ ::apply_table (const view_2d_table& table, template void Functions -::init_kokkos_tables (view_2d_table& vn_table_vals, view_2d_table& vm_table_vals, - view_2d_table& revap_table_vals, view_1d_table& mu_r_table_vals, - view_dnu_table& dnu) { - // initialize on host - - using DeviceTable1 = typename view_1d_table::non_const_type; - using DeviceTable2 = typename view_2d_table::non_const_type; - using DeviceDnuTable = typename view_dnu_table::non_const_type; - - const auto vn_table_vals_d = DeviceTable2("vn_table_vals"); - const auto vm_table_vals_d = DeviceTable2("vm_table_vals"); - const auto revap_table_vals_d = DeviceTable2("revap_table_vals"); - const auto mu_r_table_vals_d = DeviceTable1("mu_r_table_vals"); - const auto dnu_table_d = DeviceDnuTable("dnu"); - const auto vn_table_vals_h = Kokkos::create_mirror_view(vn_table_vals_d); - const auto vm_table_vals_h = Kokkos::create_mirror_view(vm_table_vals_d); - const auto revap_table_vals_h = Kokkos::create_mirror_view(revap_table_vals_d); - const auto mu_table_h = Kokkos::create_mirror_view(mu_r_table_vals_d); - const auto dnu_table_h = Kokkos::create_mirror_view(dnu_table_d); - - // Need 2d-tables with fortran-style layout - using P3F = Functions; - using LHostTable2 = typename P3F::KT::template lview; - LHostTable2 vn_table_vals_lh("vn_table_vals_lh"), vm_table_vals_lh("vm_table_vals_lh"), revap_table_vals_lh("revap_table_vals_lh"); - init_tables_from_f90_c(vn_table_vals_lh.data(), vm_table_vals_lh.data(), revap_table_vals_lh.data(), mu_table_h.data()); - for (int i = 0; i < C::VTABLE_DIM0; ++i) { - for (int j = 0; j < C::VTABLE_DIM1; ++j) { - vn_table_vals_h(i, j) = vn_table_vals_lh(i, j); - vm_table_vals_h(i, j) = vm_table_vals_lh(i, j); - revap_table_vals_h(i, j) = revap_table_vals_lh(i, j); - } - } - - dnu_table_h(0) = 0.000; - dnu_table_h(1) = -0.557; - dnu_table_h(2) = -0.430; - dnu_table_h(3) = -0.307; - dnu_table_h(4) = -0.186; - dnu_table_h(5) = -0.067; - dnu_table_h(6) = -0.050; - dnu_table_h(7) = -0.167; - dnu_table_h(8) = -0.282; - dnu_table_h(9) = -0.397; - dnu_table_h(10) = -0.512; - dnu_table_h(11) = -0.626; - dnu_table_h(12) = -0.739; - dnu_table_h(13) = -0.853; - dnu_table_h(14) = -0.966; - dnu_table_h(15) = -0.966; - - // deep copy to device - Kokkos::deep_copy(vn_table_vals_d, vn_table_vals_h); - Kokkos::deep_copy(vm_table_vals_d, vm_table_vals_h); - Kokkos::deep_copy(revap_table_vals_d, revap_table_vals_h); - Kokkos::deep_copy(mu_r_table_vals_d, mu_table_h); - Kokkos::deep_copy(dnu_table_d, dnu_table_h); - vn_table_vals = vn_table_vals_d; - vm_table_vals = vm_table_vals_d; - revap_table_vals = revap_table_vals_d; - mu_r_table_vals = mu_r_table_vals_d; - dnu = dnu_table_d; +::get_global_tables (view_2d_table& vn_table_vals, view_2d_table& vm_table_vals, + view_2d_table& revap_table_vals, view_1d_table& mu_r_table_vals, + view_dnu_table& dnu) { + auto tables = p3_init(); + vn_table_vals = tables.vn_table_vals; + vm_table_vals = tables.vm_table_vals; + revap_table_vals = tables.revap_table_vals; + mu_r_table_vals = tables.mu_r_table_vals; + dnu = tables.dnu_table_vals; } } // namespace p3 diff --git a/components/eamxx/src/physics/p3/impl/p3_table_ice_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_table_ice_impl.hpp index 4c2a43603da7..9b28a4988bc3 100644 --- a/components/eamxx/src/physics/p3/impl/p3_table_ice_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_table_ice_impl.hpp @@ -15,66 +15,10 @@ namespace p3 { template void Functions -::init_kokkos_ice_lookup_tables(view_ice_table& ice_table_vals, view_collect_table& collect_table_vals) { - - using DeviceIcetable = typename view_ice_table::non_const_type; - using DeviceColtable = typename view_collect_table::non_const_type; - - const auto ice_table_vals_d = DeviceIcetable("ice_table_vals"); - const auto collect_table_vals_d = DeviceColtable("collect_table_vals"); - - const auto ice_table_vals_h = Kokkos::create_mirror_view(ice_table_vals_d); - const auto collect_table_vals_h = Kokkos::create_mirror_view(collect_table_vals_d); - - // - // read in ice microphysics table into host views - // - - std::string filename = std::string(P3C::p3_lookup_base) + std::string(P3C::p3_version); - - std::ifstream in(filename); - - // read header - std::string version, version_val; - in >> version >> version_val; - EKAT_REQUIRE_MSG(version == "VERSION", "Bad " << filename << ", expected VERSION X.Y.Z header"); - EKAT_REQUIRE_MSG(version_val == P3C::p3_version, "Bad " << filename << ", expected version " << P3C::p3_version << ", but got " << version_val); - - // read tables - double dum_s; int dum_i; // dum_s needs to be double to stream correctly - for (int jj = 0; jj < P3C::densize; ++jj) { - for (int ii = 0; ii < P3C::rimsize; ++ii) { - for (int i = 0; i < P3C::isize; ++i) { - in >> dum_i >> dum_i; - int j_idx = 0; - for (int j = 0; j < 15; ++j) { - in >> dum_s; - if (j > 1 && j != 10) { - ice_table_vals_h(jj, ii, i, j_idx++) = dum_s; - } - } - } - - for (int i = 0; i < P3C::isize; ++i) { - for (int j = 0; j < P3C::rcollsize; ++j) { - in >> dum_i >> dum_i; - int k_idx = 0; - for (int k = 0; k < 6; ++k) { - in >> dum_s; - if (k == 3 || k == 4) { - collect_table_vals_h(jj, ii, i, j, k_idx++) = std::log10(dum_s); - } - } - } - } - } - } - - // deep copy to device - Kokkos::deep_copy(ice_table_vals_d, ice_table_vals_h); - Kokkos::deep_copy(collect_table_vals_d, collect_table_vals_h); - ice_table_vals = ice_table_vals_d; - collect_table_vals = collect_table_vals_d; +::get_global_ice_lookup_tables(view_ice_table& ice_table_vals, view_collect_table& collect_table_vals) { + auto tables = p3_init(); + ice_table_vals = tables.ice_table_vals; + collect_table_vals = tables.collect_table_vals; } template diff --git a/components/eamxx/src/physics/p3/p3_functions.hpp b/components/eamxx/src/physics/p3/p3_functions.hpp index 81a492688007..f91c08da93c3 100644 --- a/components/eamxx/src/physics/p3/p3_functions.hpp +++ b/components/eamxx/src/physics/p3/p3_functions.hpp @@ -7,6 +7,7 @@ #include "ekat/ekat_pack_kokkos.hpp" #include "ekat/ekat_workspace.hpp" +#include "ekat/ekat_parameter_list.hpp" namespace scream { namespace p3 { @@ -72,7 +73,6 @@ struct Functions using KT = KokkosTypes; using C = scream::physics::Constants; - using CP3 = scream::physics::P3_Constants; template using view_1d = typename KT::template view_1d; @@ -110,8 +110,57 @@ struct Functions // Structure to store p3 runtime options struct P3Runtime { - // maximum total ice concentration (sum of all categories) (m) - Scalar max_total_ni; + + Scalar max_total_ni = 740.0e3; + Scalar autoconversion_prefactor = 1350.0; + Scalar autoconversion_qc_exponent = 2.47; + Scalar autoconversion_nc_exponent = 1.79; + Scalar autoconversion_radius = 25.0e-6; + Scalar accretion_prefactor = 67.0; + Scalar accretion_qc_exponent = 1.15; + Scalar accretion_qr_exponent = 1.15; + Scalar rain_selfcollection_prefactor = 5.78; + Scalar rain_selfcollection_breakup_diameter = 0.00028; + Scalar constant_mu_rain = 1.0; + Scalar spa_ccn_to_nc_factor = 1.0; + Scalar cldliq_to_ice_collection_factor = 0.5; + Scalar rain_to_ice_collection_factor = 1.0; + Scalar min_rime_rho = 50.0; + Scalar max_rime_rho = 900.0; + Scalar immersion_freezing_exponent = 0.65; + Scalar deposition_nucleation_exponent = 0.304; + Scalar ice_sedimentation_factor = 1.0; + bool do_ice_production = true; + bool set_cld_frac_l_to_one = false; + bool set_cld_frac_i_to_one = false; + bool set_cld_frac_r_to_one = false; + + void load_runtime_options_from_file(ekat::ParameterList& params) { + max_total_ni = params.get("max_total_ni", max_total_ni); + autoconversion_prefactor = params.get("autoconversion_prefactor", autoconversion_prefactor); + autoconversion_qc_exponent = params.get("autoconversion_qc_exponent", autoconversion_qc_exponent); + autoconversion_nc_exponent = params.get("autoconversion_nc_exponent", autoconversion_nc_exponent); + autoconversion_radius = params.get("autoconversion_radius", autoconversion_radius); + accretion_prefactor = params.get("accretion_prefactor", accretion_prefactor); + accretion_qc_exponent = params.get("accretion_qc_exponent", accretion_qc_exponent); + accretion_qr_exponent = params.get("accretion_qr_exponent", accretion_qr_exponent); + rain_selfcollection_prefactor = params.get("rain_selfcollection_prefactor", rain_selfcollection_prefactor); + rain_selfcollection_breakup_diameter = params.get("rain_selfcollection_breakup_diameter", rain_selfcollection_breakup_diameter); + constant_mu_rain = params.get("constant_mu_rain", constant_mu_rain); + spa_ccn_to_nc_factor = params.get("spa_ccn_to_nc_factor", spa_ccn_to_nc_factor); + cldliq_to_ice_collection_factor = params.get("cldliq_to_ice_collection_factor", cldliq_to_ice_collection_factor); + rain_to_ice_collection_factor = params.get("rain_to_ice_collection_factor", rain_to_ice_collection_factor); + min_rime_rho = params.get("min_rime_rho", min_rime_rho); + max_rime_rho = params.get("max_rime_rho", max_rime_rho); + immersion_freezing_exponent = params.get("immersion_freezing_exponent", immersion_freezing_exponent); + deposition_nucleation_exponent = params.get("deposition_nucleation_exponent", deposition_nucleation_exponent); + ice_sedimentation_factor = params.get("ice_sedimentation_factor", ice_sedimentation_factor); + do_ice_production = params.get("do_ice_production", do_ice_production); + set_cld_frac_l_to_one = params.get("set_cld_frac_l_to_one", set_cld_frac_l_to_one); + set_cld_frac_i_to_one = params.get("set_cld_frac_i_to_one", set_cld_frac_i_to_one); + set_cld_frac_r_to_one = params.get("set_cld_frac_r_to_one", set_cld_frac_r_to_one); + } + }; // This struct stores prognostic variables evolved by P3. @@ -306,14 +355,17 @@ struct Functions // --------- Functions --------- // - // Call from host to initialize the static table entries. - static void init_kokkos_tables( + // Call to get global tables + static void get_global_tables( view_2d_table& vn_table_vals, view_2d_table& vm_table_vals, view_2d_table& revap_table_vals, view_1d_table& mu_r_table_vals, view_dnu_table& dnu); - static void init_kokkos_ice_lookup_tables( + static void get_global_ice_lookup_tables( view_ice_table& ice_table_vals, view_collect_table& collect_table_vals); + static P3LookupTables p3_init(const bool write_tables = false, + const bool masterproc = false); + // Map (mu_r, lamr) to Table3 data. KOKKOS_FUNCTION static void lookup(const Spack& mu_r, const Spack& lamr, @@ -505,7 +557,7 @@ struct Functions const uview_1d& qr_tend, const uview_1d& nr_tend, Scalar& precip_liq_surf, - const physics::P3_Constants & p3constants); + const P3Runtime& runtime_options); #ifdef SCREAM_P3_SMALL_KERNELS static void rain_sedimentation_disp( @@ -529,7 +581,7 @@ struct Functions const uview_1d& precip_liq_surf, const uview_1d& is_nucleat_possible, const uview_1d& is_hydromet_present, - const physics::P3_Constants & p3constants); + const P3Runtime& runtime_options); #endif // TODO: comment @@ -555,7 +607,7 @@ struct Functions const uview_1d& ni_tend, const view_ice_table& ice_table_vals, Scalar& precip_ice_surf, - const physics::P3_Constants & p3constants); + const P3Runtime& runtime_options); #ifdef SCREAM_P3_SMALL_KERNELS static void ice_sedimentation_disp( @@ -580,7 +632,7 @@ struct Functions const uview_1d& precip_ice_surf, const uview_1d& is_nucleat_possible, const uview_1d& is_hydromet_present, - const physics::P3_Constants & p3constants); + const P3Runtime& runtime_options); #endif // homogeneous freezing of cloud and rain @@ -667,7 +719,7 @@ struct Functions static void get_rain_dsd2 ( const Spack& qr, Spack& nr, Spack& mu_r, Spack& lamr, - const physics::P3_Constants & p3constants, + const P3Runtime& runtime_options, const Smask& context = Smask(true) ); // Computes and returns additional rain size distribution parameters @@ -690,7 +742,7 @@ struct Functions static void cldliq_immersion_freezing(const Spack& T_atm, const Spack& lamc, const Spack& mu_c, const Spack& cdist1, const Spack& qc_incld, const Spack& inv_qc_relvar, Spack& qc2qi_hetero_freeze_tend, Spack& nc2ni_immers_freeze_tend, - const physics::P3_Constants & p3constants, + const P3Runtime& runtime_options, const Smask& context = Smask(true) ); // Computes the immersion freezing of rain @@ -698,7 +750,7 @@ struct Functions static void rain_immersion_freezing(const Spack& T_atm, const Spack& lamr, const Spack& mu_r, const Spack& cdistr, const Spack& qr_incld, Spack& qr2qi_immers_freeze_tend, Spack& nr2ni_immers_freeze_tend, - const physics::P3_Constants & p3constants, + const P3Runtime& runtime_options, const Smask& context = Smask(true) ); // Computes droplet self collection @@ -713,7 +765,7 @@ struct Functions static void cloud_rain_accretion(const Spack& rho, const Spack& inv_rho, const Spack& qc_incld, const Spack& nc_incld, const Spack& qr_incld, const Spack& inv_qc_relvar, Spack& qc2qr_accret_tend, Spack& nc_accret_tend, - const physics::P3_Constants & p3constants, + const P3Runtime& runtime_options, const Smask& context = Smask(true) ); // Computes cloud water autoconversion process rate @@ -721,13 +773,13 @@ struct Functions static void cloud_water_autoconversion(const Spack& rho, const Spack& qc_incld, const Spack& nc_incld, const Spack& inv_qc_relvar, Spack& qc2qr_autoconv_tend, Spack& nc2nr_autoconv_tend, Spack& ncautr, - const physics::P3_Constants & p3constants, + const P3Runtime& runtime_options, const Smask& context = Smask(true)); // Computes rain self collection process rate KOKKOS_FUNCTION static void rain_self_collection(const Spack& rho, const Spack& qr_incld, const Spack& nr_incld, Spack& nr_selfcollect_tend, - const physics::P3_Constants & p3constants, + const P3Runtime& runtime_options, const Smask& context = Smask(true) ); // Impose maximum ice number @@ -742,7 +794,7 @@ struct Functions KOKKOS_FUNCTION static Spack calc_bulk_rho_rime( const Spack& qi_tot, Spack& qi_rim, Spack& bi_rim, - const physics::P3_Constants & p3constants, + const P3Runtime& runtime_options, const Smask& context = Smask(true) ); // TODO - comment @@ -751,7 +803,7 @@ struct Functions const view_2d_table& vn_table_vals, const view_2d_table& vm_table_vals, const Spack& qr_incld, const Spack& rhofacr, Spack& nr_incld, Spack& mu_r, Spack& lamr, Spack& V_qr, Spack& V_nr, - const physics::P3_Constants & p3constants, + const P3Runtime& runtime_options, const Smask& context = Smask(true)); //--------------------------------------------------------------------------------- @@ -786,7 +838,7 @@ struct Functions const Spack& qi_incld, const Spack& qc_incld, const Spack& ni_incld, const Spack& nc_incld, Spack& qc2qi_collect_tend, Spack& nc_collect_tend, Spack& qc2qr_ice_shed_tend, Spack& ncshdc, - const physics::P3_Constants & p3constants, + const P3Runtime& runtime_options, const Smask& context = Smask(true)); // TODO (comments) @@ -797,7 +849,7 @@ struct Functions const Spack& qi_incld, const Spack& ni_incld, const Spack& qr_incld, Spack& qr2qi_collect_tend, Spack& nr_collect_tend, - const physics::P3_Constants & p3constants, + const P3Runtime& runtime_options, const Smask& context = Smask(true)); // TODO (comments) @@ -882,7 +934,7 @@ struct Functions const Spack& qv_supersat_i, const Scalar& inv_dt, const bool& do_predict_nc, const bool& do_prescribed_CCN, Spack& qv2qi_nucleat_tend, Spack& ni_nucleat_tend, - const physics::P3_Constants & p3constants, + const P3Runtime& runtime_options, const Smask& context = Smask(true)); KOKKOS_FUNCTION @@ -1016,7 +1068,7 @@ struct Functions const uview_1d& bm_incld, bool& is_nucleat_possible, bool& is_hydromet_present, - const physics::P3_Constants & p3constants); + const P3Runtime& runtime_options); #ifdef SCREAM_P3_SMALL_KERNELS static void p3_main_part1_disp( @@ -1064,7 +1116,7 @@ struct Functions const uview_2d& bm_incld, const uview_1d& is_nucleat_possible, const uview_1d& is_hydromet_present, - const physics::P3_Constants & p3constants); + const P3Runtime& runtime_options); #endif KOKKOS_FUNCTION @@ -1143,7 +1195,7 @@ struct Functions const uview_1d& prctot, bool& is_hydromet_present, const Int& nk, - const physics::P3_Constants & p3constants); + const P3Runtime& runtime_options); #ifdef SCREAM_P3_SMALL_KERNELS static void p3_main_part2_disp( @@ -1221,7 +1273,7 @@ struct Functions const uview_2d& prctot, const uview_1d& is_nucleat_possible, const uview_1d& is_hydromet_present, - const physics::P3_Constants & p3constants); + const P3Runtime& runtime_options); #endif KOKKOS_FUNCTION @@ -1263,7 +1315,7 @@ struct Functions const uview_1d& diag_equiv_reflectivity, const uview_1d& diag_eff_radius_qc, const uview_1d& diag_eff_radius_qr, - const physics::P3_Constants & p3constants); + const P3Runtime& runtime_options); #ifdef SCREAM_P3_SMALL_KERNELS static void p3_main_part3_disp( @@ -1306,7 +1358,7 @@ struct Functions const uview_2d& diag_eff_radius_qr, const uview_1d& is_nucleat_possible, const uview_1d& is_hydromet_present, - const physics::P3_Constants & p3constants); + const P3Runtime& runtime_options); #endif // Return microseconds elapsed @@ -1323,8 +1375,7 @@ struct Functions #endif const WorkspaceManager& workspace_mgr, Int nj, // number of columns - Int nk, // number of vertical cells per column - const physics::P3_Constants & p3constants); + Int nk); // number of vertical cells per column static Int p3_main_internal( const P3Runtime& runtime_options, @@ -1336,8 +1387,7 @@ struct Functions const P3LookupTables& lookup_tables, const WorkspaceManager& workspace_mgr, Int nj, // number of columns - Int nk, // number of vertical cells per column - const physics::P3_Constants & p3constants); + Int nk); // number of vertical cells per column #ifdef SCREAM_P3_SMALL_KERNELS static Int p3_main_internal_disp( @@ -1351,8 +1401,7 @@ struct Functions const P3Temporaries& temporaries, const WorkspaceManager& workspace_mgr, Int nj, // number of columns - Int nk, // number of vertical cells per column - const physics::P3_Constants & p3constants); + Int nk); // number of vertical cells per column #endif KOKKOS_FUNCTION @@ -1374,13 +1423,6 @@ struct Functions template constexpr ScalarT Functions::P3C::lookup_table_1a_dum1_c; -extern "C" { -// decl of fortran function for loading tables from fortran p3. This will -// continue to be a bit awkward until we have fully ported all of p3. -void init_tables_from_f90_c(Real* vn_table_vals_data, Real* vm_table_vals_data, - Real* revap_table_vals_data, Real* mu_table_data); -} - } // namespace p3 } // namespace scream @@ -1428,5 +1470,6 @@ void init_tables_from_f90_c(Real* vn_table_vals_data, Real* vm_table_vals_data, # include "p3_nr_conservation_impl.hpp" # include "p3_ni_conservation_impl.hpp" # include "p3_prevent_liq_supersaturation_impl.hpp" +# include "p3_init_impl.hpp" #endif // GPU && !KOKKOS_ENABLE_*_RELOCATABLE_DEVICE_CODE #endif // P3_FUNCTIONS_HPP diff --git a/components/eamxx/src/physics/p3/p3_iso_c.f90 b/components/eamxx/src/physics/p3/p3_iso_c.f90 deleted file mode 100644 index ea0a18411c10..000000000000 --- a/components/eamxx/src/physics/p3/p3_iso_c.f90 +++ /dev/null @@ -1,988 +0,0 @@ -module p3_iso_c - use iso_c_binding - implicit none - -#include "scream_config.f" -#ifdef SCREAM_DOUBLE_PRECISION -# define c_real c_double -#else -# define c_real c_float -#endif - -! -! This file contains bridges from scream c++ to micro_p3 fortran. -! - -contains - subroutine append_precision(string, prefix) - - character(kind=c_char, len=512), intent(out) :: string - character(*), intent(in) :: prefix - real(kind=c_real) :: s - - write (string, '(a,i1,a1)') prefix, sizeof(s), C_NULL_CHAR - end subroutine append_precision - - subroutine init_tables_from_f90_c(vn_table_vals_c, vm_table_vals_c, revap_table_vals_c, mu_table_c) bind(C) - use micro_p3, only: p3_get_tables - - real(kind=c_real), intent(inout), dimension(300,10) :: vn_table_vals_c, vm_table_vals_c, revap_table_vals_c - real(kind=c_real), intent(inout), dimension(150) :: mu_table_c - - real(kind=c_real), dimension(150), target :: mu_table_f - real(kind=c_real), dimension(300,10), target :: vn_table_vals_f, vm_table_vals_f, revap_table_vals_f - - call p3_get_tables(mu_table_f, revap_table_vals_f, vn_table_vals_f, vm_table_vals_f) - vn_table_vals_c(:,:) = vn_table_vals_f(:,:) - vm_table_vals_c(:,:) = vm_table_vals_f(:,:) - revap_table_vals_c(:,:) = revap_table_vals_f(:,:) - mu_table_c(:) = mu_table_f(:) - - end subroutine init_tables_from_f90_c - - subroutine p3_init_c(lookup_file_dir_c, info, write_tables) bind(c) - use ekat_array_io_mod, only: array_io_file_exists -#ifdef SCREAM_DOUBLE_PRECISION - use ekat_array_io_mod, only: array_io_read=>array_io_read_double, array_io_write=>array_io_write_double -#else - use ekat_array_io_mod, only: array_io_read=>array_io_read_float, array_io_write=>array_io_write_float -#endif - use micro_p3, only: p3_init_a, p3_init_b, p3_set_tables, p3_get_tables - - type(c_ptr), intent(in) :: lookup_file_dir_c - integer(kind=c_int), intent(out) :: info - logical(kind=c_bool), intent(in) :: write_tables - - real(kind=c_real), dimension(150), target :: mu_r_table_vals - real(kind=c_real), dimension(300,10), target :: vn_table_vals, vm_table_vals, revap_table_vals - - character(len=256), pointer :: lookup_file_dir - character(kind=c_char, len=512) :: mu_r_filename, revap_filename, vn_filename, vm_filename - integer :: len - logical :: ok - character(len=16) :: p3_version="4.1.1" ! TODO: Change to be dependent on table version and path specified in p3_functions.hpp - - call c_f_pointer(lookup_file_dir_c, lookup_file_dir) - len = index(lookup_file_dir, C_NULL_CHAR) - 1 - call p3_init_a(lookup_file_dir(1:len),p3_version) - - info = 0 - ok = .false. - -#ifdef SCREAM_DOUBLE_PRECISION - mu_r_filename = lookup_file_dir(1:len)//'/mu_r_table_vals.dat8'//C_NULL_CHAR - revap_filename = lookup_file_dir(1:len)//'/revap_table_vals.dat8'//C_NULL_CHAR - vn_filename = lookup_file_dir(1:len)//'/vn_table_vals.dat8'//C_NULL_CHAR - vm_filename = lookup_file_dir(1:len)//'/vm_table_vals.dat8'//C_NULL_CHAR -#else - mu_r_filename = lookup_file_dir(1:len)//'/mu_r_table_vals.dat4'//C_NULL_CHAR - revap_filename = lookup_file_dir(1:len)//'/revap_table_vals.dat4'//C_NULL_CHAR - vn_filename = lookup_file_dir(1:len)//'/vn_table_vals.dat4'//C_NULL_CHAR - vm_filename = lookup_file_dir(1:len)//'/vm_table_vals.dat4'//C_NULL_CHAR -#endif - - if (write_tables) then - call p3_init_b() - call p3_get_tables(mu_r_table_vals, revap_table_vals, vn_table_vals, vm_table_vals) - ok = array_io_write(mu_r_filename, c_loc(mu_r_table_vals), size(mu_r_table_vals)) .and. & - array_io_write(revap_filename, c_loc(revap_table_vals), size(revap_table_vals)) .and. & - array_io_write(vn_filename, c_loc(vn_table_vals), size(vn_table_vals)) .and. & - array_io_write(vm_filename, c_loc(vm_table_vals), size(vm_table_vals)) - if (.not. ok) then - print *, 'p3_iso_c::p3_init: Error when writing table files.' - info = -1 - end if - else - ! Check table files exist - ok = array_io_file_exists(mu_r_filename) .and. & - array_io_file_exists(revap_filename) .and. & - array_io_file_exists(vn_filename) .and. & - array_io_file_exists(vm_filename) - if (.not. ok) then - print *, 'p3_iso_c::p3_init: One or more table files does not exist' - info = -2 - return - endif - - ! Read files - if (.not. array_io_read(mu_r_filename, c_loc(mu_r_table_vals), size(mu_r_table_vals))) then - print *, "p3_iso_c::p3_init: error reading mu_r table from file "//mu_r_filename - info = -3 - return - elseif (.not. array_io_read(revap_filename, c_loc(revap_table_vals), size(revap_table_vals))) then - print *, "p3_iso_c::p3_init: error reading revap table from file "//revap_filename - info = -4 - return - - elseif (.not. array_io_read(vn_filename, c_loc(vn_table_vals), size(vn_table_vals))) then - print *, "p3_iso_c::p3_init: error reading vn table from file "//vn_filename - info = -5 - return - elseif (.not. array_io_read(vm_filename, c_loc(vm_table_vals), size(vm_table_vals))) then - print *, "p3_iso_c::p3_init: error reading vm table from file "//vm_filename - info = -6 - return - endif - - call p3_set_tables(mu_r_table_vals, revap_table_vals, vn_table_vals, vm_table_vals) - end if - - end subroutine p3_init_c - - subroutine p3_main_c(qc,nc,qr,nr,th_atm,qv,dt,qi,qm,ni,bm, & - pres,dz,nc_nuceat_tend,nccn_prescribed,ni_activated,inv_qc_relvar,it,precip_liq_surf,precip_ice_surf,its,ite,kts,kte,diag_eff_radius_qc, & - diag_eff_radius_qi,diag_eff_radius_qr,rho_qi,do_predict_nc,do_prescribed_CCN,dpres,inv_exner,qv2qi_depos_tend, & - precip_liq_flux,precip_ice_flux,cld_frac_r,cld_frac_l,cld_frac_i,liq_ice_exchange, & - vap_liq_exchange, vap_ice_exchange, qv_prev, t_prev, elapsed_s) bind(C) - use micro_p3, only : p3_main - - real(kind=c_real), intent(inout), dimension(its:ite,kts:kte) :: qc, nc, qr, nr, qv, th_atm - real(kind=c_real), intent(inout), dimension(its:ite,kts:kte) :: qi, qm, ni, bm - real(kind=c_real), intent(in), dimension(its:ite,kts:kte) :: pres, dz - real(kind=c_real), intent(in), dimension(its:ite,kts:kte) :: nc_nuceat_tend,nccn_prescribed,ni_activated - real(kind=c_real), intent(in), dimension(its:ite,kts:kte) :: inv_qc_relvar - real(kind=c_real), value, intent(in) :: dt - real(kind=c_real), intent(out), dimension(its:ite) :: precip_liq_surf, precip_ice_surf - real(kind=c_real), intent(out), dimension(its:ite,kts:kte) :: diag_eff_radius_qc - real(kind=c_real), intent(out), dimension(its:ite,kts:kte) :: diag_eff_radius_qi - real(kind=c_real), intent(out), dimension(its:ite,kts:kte) :: diag_eff_radius_qr - real(kind=c_real), intent(out), dimension(its:ite,kts:kte) :: rho_qi - integer(kind=c_int), value, intent(in) :: its,ite, kts,kte, it - logical(kind=c_bool), value, intent(in) :: do_predict_nc,do_prescribed_CCN - - real(kind=c_real), intent(in), dimension(its:ite,kts:kte) :: dpres - real(kind=c_real), intent(in), dimension(its:ite,kts:kte) :: inv_exner - real(kind=c_real), intent(out), dimension(its:ite,kts:kte) :: qv2qi_depos_tend - real(kind=c_real), intent(out), dimension(its:ite,kts:kte+1) :: precip_liq_flux - real(kind=c_real), intent(out), dimension(its:ite,kts:kte+1) :: precip_ice_flux - real(kind=c_real), intent(in), dimension(its:ite,kts:kte) :: cld_frac_i, cld_frac_l, cld_frac_r - real(kind=c_real), intent(out), dimension(its:ite,kts:kte) :: liq_ice_exchange - real(kind=c_real), intent(out), dimension(its:ite,kts:kte) :: vap_liq_exchange - real(kind=c_real), intent(out), dimension(its:ite,kts:kte) :: vap_ice_exchange - real(kind=c_real), intent(in), dimension(its:ite,kts:kte) :: qv_prev - real(kind=c_real), intent(in), dimension(its:ite,kts:kte) :: t_prev - - real(kind=c_real), intent(out) :: elapsed_s - - real(kind=c_real), dimension(its:ite,kts:kte,49) :: p3_tend_out - real(kind=c_real), dimension(its:ite,3) :: col_location - real(kind=c_real), dimension(its:ite,kts:kte) :: mu_c, lamc - real(kind=c_real), dimension(its:ite,kts:kte) :: precip_total_tend - real(kind=c_real), dimension(its:ite,kts:kte) :: nevapr - real(kind=c_real), dimension(its:ite,kts:kte) :: qr_evap_tend - integer :: i - do i = its,ite - col_location(i,:) = real(i) - end do - - call p3_main(qc,nc,qr,nr,th_atm,qv,dt,qi,qm,ni,bm, & - pres,dz,nc_nuceat_tend,nccn_prescribed,ni_activated,inv_qc_relvar,it,precip_liq_surf,precip_ice_surf,its,ite,kts,kte,diag_eff_radius_qc, & - diag_eff_radius_qi,diag_eff_radius_qr, rho_qi,do_predict_nc,do_prescribed_CCN,dpres,inv_exner,qv2qi_depos_tend,precip_total_tend,nevapr, & - qr_evap_tend,precip_liq_flux,precip_ice_flux,cld_frac_r,cld_frac_l,cld_frac_i,p3_tend_out,mu_c,lamc,liq_ice_exchange,& - vap_liq_exchange,vap_ice_exchange,qv_prev,t_prev,col_location,elapsed_s) - end subroutine p3_main_c - - subroutine micro_p3_utils_init_c(Cpair, Rair, RH2O, RHO_H2O, & - MWH2O, MWdry, gravit, LatVap, LatIce, & - CpLiq, Tmelt, Pi, masterproc) bind(C) - - use micro_p3_utils, only: micro_p3_utils_init - use iso_fortran_env, only: OUTPUT_UNIT - real(kind=c_real), value, intent(in) :: Cpair - real(kind=c_real), value, intent(in) :: Rair - real(kind=c_real), value, intent(in) :: RH2O - real(kind=c_real), value, intent(in) :: RHO_H2O - real(kind=c_real), value, intent(in) :: MWH2O - real(kind=c_real), value, intent(in) :: MWdry - real(kind=c_real), value, intent(in) :: gravit - real(kind=c_real), value, intent(in) :: LatVap - real(kind=c_real), value, intent(in) :: LatIce - real(kind=c_real), value, intent(in) :: CpLiq - real(kind=c_real), value, intent(in) :: Tmelt - real(kind=c_real), value, intent(in) :: Pi - logical(kind=c_bool), value, intent(in) :: masterproc - - call micro_p3_utils_init(Cpair,Rair,RH2O,RHO_H2O,MWH2O,MWdry,gravit,LatVap,LatIce, & - CpLiq,Tmelt,Pi,OUTPUT_UNIT,masterproc) - end subroutine micro_p3_utils_init_c - - subroutine p3_init_a_c(ice_table_vals_c, collect_table_vals_c) bind(C) - use micro_p3, only: ice_table_vals, collect_table_vals - use micro_p3_utils, only: densize,rimsize,isize,ice_table_size,rcollsize,collect_table_size - - real(kind=c_real), intent(out), dimension(densize,rimsize,isize,ice_table_size) :: ice_table_vals_c - real(kind=c_real), intent(out), dimension(densize,rimsize,isize,rcollsize,collect_table_size) :: collect_table_vals_c - - ice_table_vals_c(:,:,:,:) = ice_table_vals(:,:,:,:) - collect_table_vals_c(:,:,:,:,:) = collect_table_vals(:,:,:,:,:) - end subroutine p3_init_a_c - - subroutine find_lookuptable_indices_1a_c(dumi,dumjj,dumii,dumzz,dum1,dum4,dum5,dum6, & - qi,ni,qm,rhop) bind(C) - use micro_p3, only: find_lookupTable_indices_1a - use micro_p3_utils, only: densize,rimsize,isize - - ! arguments: - integer(kind=c_int), intent(out) :: dumi,dumjj,dumii,dumzz - real(kind=c_real), intent(out) :: dum1,dum4,dum5,dum6 - real(kind=c_real), value, intent(in) :: qi,ni,qm,rhop - - call find_lookupTable_indices_1a(dumi, dumjj, dumii, dumzz, dum1, dum4, dum5, dum6, & - isize, rimsize, densize, qi, ni, qm, rhop) - end subroutine find_lookuptable_indices_1a_c - - subroutine find_lookuptable_indices_1b_c(dumj,dum3,qr,nr) bind(C) - use micro_p3, only: find_lookupTable_indices_1b - use micro_p3_utils, only: rcollsize - - integer(kind=c_int), intent(out) :: dumj - real(kind=c_real), intent(out) :: dum3 - real(kind=c_real), value, intent(in) :: qr, nr - - call find_lookupTable_indices_1b(dumj, dum3, rcollsize, qr, nr) - end subroutine find_lookupTable_indices_1b_c - - subroutine access_lookup_table_c(dumjj,dumii,dumi,index,dum1,dum4,dum5,proc) bind(C) - use micro_p3, only: access_lookup_table - - integer(kind=c_int), value, intent(in) :: dumjj, dumii, dumi, index - real(kind=c_real), value, intent(in) :: dum1, dum4, dum5 - real(kind=c_real), intent(out) :: proc - - call access_lookup_table(dumjj,dumii,dumi,index,dum1,dum4,dum5,proc) - end subroutine access_lookup_table_c - - subroutine access_lookup_table_coll_c(dumjj,dumii,dumj,dumi,index,dum1,dum3,dum4,dum5,proc) bind(C) - use micro_p3, only: access_lookup_table_coll - - integer(kind=c_int), value, intent(in) :: dumjj,dumii,dumj,dumi,index - real(kind=c_real), value, intent(in) :: dum1,dum3,dum4,dum5 - real(kind=c_real), intent(out) :: proc - - call access_lookup_table_coll(dumjj,dumii,dumj,dumi,index,dum1,dum3,dum4,dum5,proc) - end subroutine access_lookup_table_coll_c - - subroutine back_to_cell_average_c(cld_frac_l,cld_frac_r,cld_frac_i, qc2qr_accret_tend,qr2qv_evap_tend,qc2qr_autoconv_tend,& - nc_accret_tend,nc_selfcollect_tend,nc2nr_autoconv_tend,nr_selfcollect_tend,nr_evap_tend,ncautr,qi2qv_sublim_tend,nr_ice_shed_tend,qc2qi_hetero_freeze_tend,& - qr2qi_collect_tend,qc2qr_ice_shed_tend,qi2qr_melt_tend,qc2qi_collect_tend,qr2qi_immers_freeze_tend,ni2nr_melt_tend,nc_collect_tend,ncshdc,nc2ni_immers_freeze_tend,nr_collect_tend,ni_selfcollect_tend,& - qv2qi_vapdep_tend,nr2ni_immers_freeze_tend,ni_sublim_tend,qv2qi_nucleat_tend,ni_nucleat_tend,qc2qi_berg_tend) bind(C) - - use micro_p3, only: back_to_cell_average - real(kind=c_real), value, intent(in) :: cld_frac_l, cld_frac_r, cld_frac_i - - real(kind=c_real), intent(inout) :: qc2qr_accret_tend, qr2qv_evap_tend, qc2qr_autoconv_tend, nc_accret_tend, nc_selfcollect_tend, nc2nr_autoconv_tend, & - nr_selfcollect_tend, nr_evap_tend, ncautr, qi2qv_sublim_tend, & - nr_ice_shed_tend, qc2qi_hetero_freeze_tend, qr2qi_collect_tend, qc2qr_ice_shed_tend, qi2qr_melt_tend, qc2qi_collect_tend, & - qr2qi_immers_freeze_tend, ni2nr_melt_tend, nc_collect_tend, ncshdc, nc2ni_immers_freeze_tend, nr_collect_tend,& - ni_selfcollect_tend, qv2qi_vapdep_tend, nr2ni_immers_freeze_tend, ni_sublim_tend, qv2qi_nucleat_tend, ni_nucleat_tend, & - qc2qi_berg_tend - - call back_to_cell_average(cld_frac_l, cld_frac_r, cld_frac_i, qc2qr_accret_tend, qr2qv_evap_tend, qc2qr_autoconv_tend,& - nc_accret_tend, nc_selfcollect_tend, nc2nr_autoconv_tend, nr_selfcollect_tend, nr_evap_tend, ncautr, qi2qv_sublim_tend, nr_ice_shed_tend, qc2qi_hetero_freeze_tend,& - qr2qi_collect_tend, qc2qr_ice_shed_tend, qi2qr_melt_tend, qc2qi_collect_tend, qr2qi_immers_freeze_tend, ni2nr_melt_tend, nc_collect_tend, ncshdc, nc2ni_immers_freeze_tend, nr_collect_tend, ni_selfcollect_tend,& - qv2qi_vapdep_tend, nr2ni_immers_freeze_tend, ni_sublim_tend, qv2qi_nucleat_tend, ni_nucleat_tend, qc2qi_berg_tend) - end subroutine back_to_cell_average_c - - subroutine cloud_water_conservation_c(qc,dt,qc2qr_autoconv_tend,qc2qr_accret_tend,qc2qi_collect_tend,qc2qi_hetero_freeze_tend,qc2qr_ice_shed_tend, & - qc2qi_berg_tend,qi2qv_sublim_tend,qv2qi_vapdep_tend) bind(C) - use micro_p3, only: cloud_water_conservation - - real(kind=c_real), value, intent(in) :: qc, dt - real(kind=c_real), intent(inout) :: qc2qr_autoconv_tend, qc2qr_accret_tend, qc2qi_collect_tend, qc2qi_hetero_freeze_tend, qc2qr_ice_shed_tend, qc2qi_berg_tend, qi2qv_sublim_tend, qv2qi_vapdep_tend - - call cloud_water_conservation(qc,dt,qc2qr_autoconv_tend,qc2qr_accret_tend,qc2qi_collect_tend,qc2qi_hetero_freeze_tend,qc2qr_ice_shed_tend,qc2qi_berg_tend,qi2qv_sublim_tend,qv2qi_vapdep_tend) - end subroutine cloud_water_conservation_c - - subroutine rain_water_conservation_c(qr,qc2qr_autoconv_tend,qc2qr_accret_tend,qi2qr_melt_tend,qc2qr_ice_shed_tend,dt, & - qr2qv_evap_tend,qr2qi_collect_tend,qr2qi_immers_freeze_tend) bind(C) - use micro_p3, only: rain_water_conservation - - real(kind=c_real), value, intent(in) :: qr, qc2qr_autoconv_tend, qc2qr_accret_tend, qi2qr_melt_tend, qc2qr_ice_shed_tend, dt - real(kind=c_real), intent(inout) :: qr2qv_evap_tend, qr2qi_collect_tend, qr2qi_immers_freeze_tend - - call rain_water_conservation(qr,qc2qr_autoconv_tend,qc2qr_accret_tend,qi2qr_melt_tend,qc2qr_ice_shed_tend,dt,qr2qv_evap_tend,qr2qi_collect_tend,qr2qi_immers_freeze_tend) - end subroutine rain_water_conservation_c - - subroutine rain_self_collection_c(rho, qr_incld, nr_incld, nr_selfcollect_tend) bind(C) - use micro_p3, only: rain_self_collection - - real(kind=c_real), value, intent(in) :: rho, qr_incld, nr_incld - real(kind=c_real), intent(out) :: nr_selfcollect_tend - - call rain_self_collection(rho, qr_incld, nr_incld, nr_selfcollect_tend) - end subroutine rain_self_collection_c - - subroutine ice_water_conservation_c(qi,qv2qi_vapdep_tend,qv2qi_nucleat_tend,qc2qi_berg_tend,qr2qi_collect_tend,qc2qi_collect_tend,qr2qi_immers_freeze_tend,qc2qi_hetero_freeze_tend,dt, & - qi2qv_sublim_tend,qi2qr_melt_tend) bind(C) - use micro_p3, only: ice_water_conservation - - real(kind=c_real), value, intent(in) :: qi, qv2qi_vapdep_tend, qv2qi_nucleat_tend, qr2qi_collect_tend, qc2qi_collect_tend, qr2qi_immers_freeze_tend, qc2qi_hetero_freeze_tend, qc2qi_berg_tend, dt - real(kind=c_real), intent(inout) :: qi2qv_sublim_tend, qi2qr_melt_tend - - call ice_water_conservation(qi,qv2qi_vapdep_tend,qv2qi_nucleat_tend,qr2qi_collect_tend,qc2qi_collect_tend,qr2qi_immers_freeze_tend,qc2qi_hetero_freeze_tend,qc2qi_berg_tend,dt,qi2qv_sublim_tend,qi2qr_melt_tend) - end subroutine ice_water_conservation_c - - subroutine get_cloud_dsd2_c(qc,nc,mu_c,rho,nu,lamc,cdist,cdist1) bind(C) - use micro_p3, only: get_cloud_dsd2 - use micro_p3_utils, only: dnu - - !arguments: - real(kind=c_real), value, intent(in) :: qc,rho - real(kind=c_real), intent(inout) :: nc - real(kind=c_real), intent(out) :: mu_c,nu,lamc,cdist,cdist1 - - call get_cloud_dsd2(qc,nc,mu_c,rho,nu,dnu,lamc,cdist,cdist1) - end subroutine get_cloud_dsd2_c - - subroutine get_rain_dsd2_c(qr,nr,mu_r,lamr,cdistr,logn0r) bind(C) - use micro_p3, only: get_rain_dsd2 - - !arguments: - real(kind=c_real), value, intent(in) :: qr - real(kind=c_real), intent(inout) :: nr - real(kind=c_real), intent(out) :: lamr,mu_r,cdistr,logn0r - - call get_rain_dsd2(qr,nr,mu_r,lamr,cdistr,logn0r) - end subroutine get_rain_dsd2_c - - subroutine calc_rime_density_c(T_atm,rhofaci,table_val_qi_fallspd,acn,lamc,mu_c,qc_incld,qc2qi_collect_tend, & - vtrmi1,rho_qm_cloud) bind(C) - - use micro_p3, only: calc_rime_density - real(kind=c_real), value, intent(in) :: T_atm, rhofaci, table_val_qi_fallspd, acn, lamc, mu_c, qc_incld, qc2qi_collect_tend - real(kind=c_real), intent(out) :: vtrmi1, rho_qm_cloud - - call calc_rime_density(T_atm, rhofaci, table_val_qi_fallspd, acn, lamc, mu_c, qc_incld, qc2qi_collect_tend, vtrmi1, rho_qm_cloud) - end subroutine calc_rime_density_c - - subroutine cldliq_immersion_freezing_c(T_atm,lamc,mu_c,cdist1,qc_incld,inv_qc_relvar,qc2qi_hetero_freeze_tend,nc2ni_immers_freeze_tend) bind(C) - - use micro_p3, only: cldliq_immersion_freezing - real(kind=c_real), value, intent(in) :: T_atm, lamc, mu_c, cdist1, qc_incld,inv_qc_relvar - real(kind=c_real), intent(out) :: qc2qi_hetero_freeze_tend, nc2ni_immers_freeze_tend - - call cldliq_immersion_freezing(T_atm, lamc, mu_c, cdist1, qc_incld, inv_qc_relvar, qc2qi_hetero_freeze_tend, nc2ni_immers_freeze_tend) - end subroutine cldliq_immersion_freezing_c - - subroutine rain_immersion_freezing_c(T_atm,lamr,mu_r,cdistr,qr_incld,qr2qi_immers_freeze_tend,nr2ni_immers_freeze_tend) bind(C) - - use micro_p3, only: rain_immersion_freezing - real(kind=c_real), value, intent(in) :: T_atm, lamr, mu_r, cdistr, qr_incld - real(kind=c_real), intent(out) :: qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend - - call rain_immersion_freezing(T_atm, lamr, mu_r, cdistr, qr_incld, qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend) - end subroutine rain_immersion_freezing_c - - subroutine droplet_self_collection_c(rho,inv_rho,qc_incld,mu_c,nu,nc2nr_autoconv_tend,nc_selfcollect_tend) bind(C) - - use micro_p3, only: droplet_self_collection - real(kind=c_real), value, intent(in) :: rho, inv_rho, qc_incld, mu_c, nu, nc2nr_autoconv_tend - real(kind=c_real), intent(out) :: nc_selfcollect_tend - - call droplet_self_collection(rho, inv_rho, qc_incld, mu_c, nu, nc2nr_autoconv_tend, nc_selfcollect_tend) - end subroutine droplet_self_collection_c - - subroutine cloud_rain_accretion_c(rho,inv_rho,qc_incld,nc_incld,qr_incld,inv_qc_relvar,qc2qr_accret_tend,nc_accret_tend) bind(C) - - use micro_p3, only: cloud_rain_accretion - real(kind=c_real), value, intent(in) :: rho, inv_rho, qc_incld, nc_incld, qr_incld,inv_qc_relvar - real(kind=c_real), intent(out) :: qc2qr_accret_tend, nc_accret_tend - - call cloud_rain_accretion(rho, inv_rho, qc_incld, nc_incld, qr_incld, inv_qc_relvar, qc2qr_accret_tend, nc_accret_tend) - end subroutine cloud_rain_accretion_c - - subroutine cloud_water_autoconversion_c(rho,qc_incld,nc_incld,inv_qc_relvar,qc2qr_autoconv_tend,nc2nr_autoconv_tend,ncautr) bind(C) - - use micro_p3, only: cloud_water_autoconversion - real(kind=c_real), value, intent(in) :: rho, qc_incld, nc_incld,inv_qc_relvar - real(kind=c_real), intent(inout) :: qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr - - call cloud_water_autoconversion(rho, qc_incld, nc_incld, inv_qc_relvar, qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr) - end subroutine cloud_water_autoconversion_c - - subroutine impose_max_total_ni_c(ni_local, max_total_ni, inv_rho_local) bind(C) - use micro_p3, only: impose_max_total_ni - - real(kind=c_real), intent(inout) :: ni_local - real(kind=c_real), value, intent(in) :: max_total_ni, inv_rho_local - - call impose_max_total_ni(ni_local, max_total_ni, inv_rho_local) - end subroutine impose_max_total_ni_c - - subroutine calc_first_order_upwind_step_c(kts, kte, kdir, kbot, k_qxtop, dt_sub, rho, inv_rho, inv_dz, num_arrays, fluxes, vs, qnx) bind(C) - use micro_p3, only: calc_first_order_upwind_step, realptr - - !arguments: - integer(kind=c_int), value, intent(in) :: kts, kte, kdir, kbot, k_qxtop, num_arrays - real(kind=c_real), value, intent(in) :: dt_sub - real(kind=c_real), dimension(kts:kte), intent(in) :: rho, inv_rho, inv_dz - type(c_ptr), intent(in), dimension(num_arrays) :: fluxes, vs, qnx - - type(realptr), dimension(num_arrays) :: fluxes_f, vs_f, qnx_f - integer :: i - - do i = 1, num_arrays - call c_f_pointer(fluxes(i), fluxes_f(i)%p, [(kte-kts)+1]) - call c_f_pointer(vs(i), vs_f(i)%p, [(kte-kts)+1]) - call c_f_pointer(qnx(i), qnx_f(i)%p , [(kte-kts)+1]) - end do - - call calc_first_order_upwind_step(kts, kte, kdir, kbot, k_qxtop, dt_sub, rho, inv_rho, inv_dz, num_arrays, fluxes_f, vs_f, qnx_f) - - end subroutine calc_first_order_upwind_step_c - - subroutine generalized_sedimentation_c(kts, kte, kdir, k_qxtop, k_qxbot, kbot, Co_max, dt_left, prt_accum, inv_dz, inv_rho, rho, num_arrays, vs, fluxes, qnx) bind(C) - use micro_p3, only: generalized_sedimentation, realptr - - ! arguments - integer(kind=c_int), value, intent(in) :: kts, kte, kdir, k_qxtop, kbot, num_arrays - integer(kind=c_int), intent(inout) :: k_qxbot - real(kind=c_real), value, intent(in) :: Co_max - real(kind=c_real), intent(inout) :: dt_left, prt_accum - real(kind=c_real), dimension(kts:kte), intent(in) :: inv_dz, inv_rho, rho - type(c_ptr), intent(in), dimension(num_arrays) :: vs, fluxes, qnx - - type(realptr), dimension(num_arrays) :: fluxes_f, vs_f, qnx_f - integer :: i - - do i = 1, num_arrays - call c_f_pointer(fluxes(i), fluxes_f(i)%p, [(kte-kts)+1]) - call c_f_pointer(vs(i), vs_f(i)%p, [(kte-kts)+1]) - call c_f_pointer(qnx(i), qnx_f(i)%p , [(kte-kts)+1]) - end do - - call generalized_sedimentation(kts, kte, kdir, k_qxtop, k_qxbot, kbot, Co_max, dt_left, prt_accum, inv_dz, inv_rho, rho, num_arrays, vs_f, fluxes_f, qnx_f) - - end subroutine generalized_sedimentation_c - - subroutine cloud_sedimentation_c(kts,kte,ktop,kbot,kdir, & - qc_incld,rho,inv_rho,cld_frac_l,acn,inv_dz,& - dt,inv_dt,do_predict_nc, & - qc, nc, nc_incld,mu_c,lamc,precip_liq_surf,qc_tend,nc_tend) bind(C) - use micro_p3, only: cloud_sedimentation, dnu - - ! arguments - integer(kind=c_int), value, intent(in) :: kts, kte, ktop, kbot, kdir - - real(kind=c_real), intent(in), dimension(kts:kte) :: rho - real(kind=c_real), intent(in), dimension(kts:kte) :: inv_rho - real(kind=c_real), intent(in), dimension(kts:kte) :: cld_frac_l - real(kind=c_real), intent(in), dimension(kts:kte) :: acn - real(kind=c_real), intent(in), dimension(kts:kte) :: inv_dz - - real(kind=c_real), value, intent(in) :: dt - real(kind=c_real), value, intent(in) :: inv_dt - logical(kind=c_bool), value, intent(in) :: do_predict_nc - - real(kind=c_real), intent(inout), dimension(kts:kte) :: qc - real(kind=c_real), intent(inout), dimension(kts:kte) :: nc - real(kind=c_real), intent(inout), dimension(kts:kte) :: qc_incld - real(kind=c_real), intent(inout), dimension(kts:kte) :: nc_incld - real(kind=c_real), intent(inout), dimension(kts:kte) :: mu_c - real(kind=c_real), intent(inout), dimension(kts:kte) :: lamc - real(kind=c_real), intent(inout) :: precip_liq_surf - real(kind=c_real), intent(inout), dimension(kts:kte) :: qc_tend - real(kind=c_real), intent(inout), dimension(kts:kte) :: nc_tend - - call cloud_sedimentation(kts,kte,ktop,kbot,kdir, & - qc_incld,rho,inv_rho,cld_frac_l,acn,inv_dz,& - dt,inv_dt,dnu,do_predict_nc, & - qc, nc, nc_incld,mu_c,lamc,precip_liq_surf,qc_tend,nc_tend) - - end subroutine cloud_sedimentation_c - - subroutine ice_sedimentation_c(kts,kte,ktop,kbot,kdir, & - rho,inv_rho,rhofaci,cld_frac_i,inv_dz,dt,inv_dt, & - qi,qi_incld,ni,qm,qm_incld,bm,bm_incld,ni_incld,precip_ice_surf,qi_tend,ni_tend) bind(C) - use micro_p3, only: ice_sedimentation - - ! arguments - integer(kind=c_int), value, intent(in) :: kts, kte, ktop, kbot, kdir - - real(kind=c_real), intent(in), dimension(kts:kte) :: rho - real(kind=c_real), intent(in), dimension(kts:kte) :: inv_rho - real(kind=c_real), intent(in), dimension(kts:kte) :: rhofaci - real(kind=c_real), intent(in), dimension(kts:kte) :: cld_frac_i - real(kind=c_real), intent(in), dimension(kts:kte) :: inv_dz - real(kind=c_real), value, intent(in) :: dt, inv_dt - - real(kind=c_real), intent(inout), dimension(kts:kte), target :: qi - real(kind=c_real), intent(inout), dimension(kts:kte) :: qi_incld - real(kind=c_real), intent(inout), dimension(kts:kte), target :: ni - real(kind=c_real), intent(inout), dimension(kts:kte) :: ni_incld - real(kind=c_real), intent(inout), dimension(kts:kte), target :: qm - real(kind=c_real), intent(inout), dimension(kts:kte) :: qm_incld - real(kind=c_real), intent(inout), dimension(kts:kte), target :: bm - real(kind=c_real), intent(inout), dimension(kts:kte) :: bm_incld - - real(kind=c_real), intent(inout) :: precip_ice_surf - real(kind=c_real), intent(inout), dimension(kts:kte) :: qi_tend - real(kind=c_real), intent(inout), dimension(kts:kte) :: ni_tend - - call ice_sedimentation(kts,kte,ktop,kbot,kdir, & - rho,inv_rho,rhofaci,cld_frac_i,inv_dz,dt,inv_dt, & - qi,qi_incld,ni,qm,qm_incld,bm,bm_incld,ni_incld,precip_ice_surf,qi_tend,ni_tend) - - end subroutine ice_sedimentation_c - - subroutine rain_sedimentation_c(kts,kte,ktop,kbot,kdir, & - qr_incld,rho,inv_rho,rhofacr,cld_frac_r,inv_dz,dt,inv_dt, & - qr,nr,nr_incld,mu_r,lamr,precip_liq_surf,precip_liq_flux,qr_tend,nr_tend) bind(C) - use micro_p3, only: rain_sedimentation - - integer(kind=c_int), value, intent(in) :: kts, kte, ktop, kbot, kdir - - real(kind=c_real), intent(in), dimension(kts:kte) :: rho - real(kind=c_real), intent(in), dimension(kts:kte) :: inv_rho - real(kind=c_real), intent(in), dimension(kts:kte) :: rhofacr - real(kind=c_real), intent(in), dimension(kts:kte) :: cld_frac_r - real(kind=c_real), intent(in), dimension(kts:kte) :: inv_dz - real(kind=c_real), value, intent(in) :: dt, inv_dt - - real(kind=c_real), intent(inout), target, dimension(kts:kte) :: qr - real(kind=c_real), intent(inout), target, dimension(kts:kte) :: nr - real(kind=c_real), intent(inout), dimension(kts:kte) :: qr_incld - real(kind=c_real), intent(inout), dimension(kts:kte) :: nr_incld - real(kind=c_real), intent(inout), dimension(kts:kte) :: mu_r - real(kind=c_real), intent(inout), dimension(kts:kte) :: lamr - real(kind=c_real), intent(inout) :: precip_liq_surf - real(kind=c_real), intent(inout), dimension(kts:kte+1) :: precip_liq_flux - real(kind=c_real), intent(inout), dimension(kts:kte) :: qr_tend - real(kind=c_real), intent(inout), dimension(kts:kte) :: nr_tend - - call rain_sedimentation(kts,kte,ktop,kbot,kdir, & - qr_incld,rho,inv_rho,rhofacr,cld_frac_r,inv_dz,dt,inv_dt, & - qr,nr,nr_incld,mu_r,lamr,precip_liq_surf,precip_liq_flux,qr_tend,nr_tend) - - end subroutine rain_sedimentation_c - - subroutine calc_bulk_rho_rime_c(qi_tot, qi_rim, bi_rim, rho_rime) bind(C) - use micro_p3, only: calc_bulkRhoRime - - ! arguments: - real(kind=c_real), value, intent(in) :: qi_tot - real(kind=c_real), intent(inout) :: qi_rim, bi_rim - real(kind=c_real), intent(out) :: rho_rime - - call calc_bulkRhoRime(qi_tot, qi_rim, bi_rim, rho_rime) - end subroutine calc_bulk_rho_rime_c - - subroutine homogeneous_freezing_c(kts,kte,ktop,kbot,kdir,T_atm,inv_exner,latent_heat_fusion, & - qc,nc,qr,nr,qi,ni,qm,bm,th_atm) bind(C) - use micro_p3, only: homogeneous_freezing - - ! arguments: - integer(kind=c_int), value, intent(in) :: kts, kte, ktop, kbot, kdir - real(kind=c_real), intent(in), dimension(kts:kte) :: T_atm - real(kind=c_real), intent(in), dimension(kts:kte) :: inv_exner - real(kind=c_real), intent(in), dimension(kts:kte) :: latent_heat_fusion - - real(kind=c_real), intent(inout), dimension(kts:kte) :: qc - real(kind=c_real), intent(inout), dimension(kts:kte) :: nc - real(kind=c_real), intent(inout), dimension(kts:kte) :: qr - real(kind=c_real), intent(inout), dimension(kts:kte) :: nr - - real(kind=c_real), intent(inout), dimension(kts:kte) :: qi - real(kind=c_real), intent(inout), dimension(kts:kte) :: ni - real(kind=c_real), intent(inout), dimension(kts:kte) :: qm - real(kind=c_real), intent(inout), dimension(kts:kte) :: bm - real(kind=c_real), intent(inout), dimension(kts:kte) :: th_atm - - call homogeneous_freezing(kts,kte,ktop,kbot,kdir,T_atm,inv_exner,latent_heat_fusion, & - qc,nc,qr,nr,qi,ni,qm,bm,th_atm) - end subroutine homogeneous_freezing_c - - subroutine compute_rain_fall_velocity_c(qr_incld, rhofacr, nr_incld, mu_r, lamr, V_qr, V_nr) bind(C) - use micro_p3, only: compute_rain_fall_velocity - - ! arguments: - real(kind=c_real), value, intent(in) :: qr_incld, rhofacr - real(kind=c_real), intent(inout) :: nr_incld - real(kind=c_real), intent(out) :: mu_r, lamr, V_qr, V_nr - - call compute_rain_fall_velocity(qr_incld, rhofacr, nr_incld, mu_r, lamr, V_qr, V_nr) - end subroutine compute_rain_fall_velocity_c - -subroutine update_prognostic_ice_c(qc2qi_hetero_freeze_tend,qc2qi_collect_tend,qc2qr_ice_shed_tend,nc_collect_tend,nc2ni_immers_freeze_tend,ncshdc,qr2qi_collect_tend,nr_collect_tend,qr2qi_immers_freeze_tend,nr2ni_immers_freeze_tend,nr_ice_shed_tend, & - qi2qr_melt_tend,ni2nr_melt_tend,qi2qv_sublim_tend,qv2qi_vapdep_tend,qv2qi_nucleat_tend,ni_nucleat_tend,ni_selfcollect_tend,ni_sublim_tend,qc2qi_berg_tend,inv_exner,latent_heat_sublim,latent_heat_fusion,do_predict_nc,log_wetgrowth, & - dt,nmltratio,rho_qm_cloud,th_atm,qv,qi,ni,qm,bm,qc,nc,qr,nr) bind(C) - use micro_p3, only: update_prognostic_ice - - ! arguments - real(kind=c_real), value, intent(in) :: qc2qi_hetero_freeze_tend, qc2qi_collect_tend, qc2qr_ice_shed_tend, nc_collect_tend, nc2ni_immers_freeze_tend, ncshdc, qr2qi_collect_tend, nr_collect_tend, & - qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend, nr_ice_shed_tend, qi2qr_melt_tend, ni2nr_melt_tend, qi2qv_sublim_tend, qv2qi_vapdep_tend, qv2qi_nucleat_tend, ni_nucleat_tend, ni_selfcollect_tend, ni_sublim_tend, qc2qi_berg_tend, inv_exner, & - latent_heat_fusion, latent_heat_sublim, dt, nmltratio, rho_qm_cloud - - logical(kind=c_bool), value, intent(in) :: do_predict_nc, log_wetgrowth - - real(kind=c_real), intent(inout) :: th_atm, qv, qc, nc, qr, nr, qi, ni, qm, bm - - call update_prognostic_ice(qc2qi_hetero_freeze_tend,qc2qi_collect_tend,qc2qr_ice_shed_tend,nc_collect_tend,nc2ni_immers_freeze_tend,ncshdc,qr2qi_collect_tend,nr_collect_tend,qr2qi_immers_freeze_tend,nr2ni_immers_freeze_tend,nr_ice_shed_tend, & - qi2qr_melt_tend,ni2nr_melt_tend,qi2qv_sublim_tend,qv2qi_vapdep_tend,qv2qi_nucleat_tend,ni_nucleat_tend,ni_selfcollect_tend,ni_sublim_tend,qc2qi_berg_tend,inv_exner,latent_heat_sublim,latent_heat_fusion,do_predict_nc,log_wetgrowth, & - dt,nmltratio,rho_qm_cloud,th_atm,qv,qi,ni,qm,bm,qc,nc,qr,nr) - - end subroutine update_prognostic_ice_c - - subroutine get_time_space_phys_variables_c(T_atm, pres, rho, latent_heat_vapor, latent_heat_sublim, qv_sat_l, qv_sat_i, mu, dv, sc, dqsdt, dqsidt, & - ab, abi, kap, eii) bind(C) - use micro_p3, only: get_time_space_phys_variables - - !arguments - real(kind=c_real), value, intent(in) :: T_atm, pres, rho, latent_heat_vapor, latent_heat_sublim, qv_sat_l, qv_sat_i - real(kind=c_real), intent(out) :: mu, dv, sc, dqsdt, dqsidt, ab, abi, kap, eii - - call get_time_space_phys_variables(T_atm, pres, rho, latent_heat_vapor, latent_heat_sublim, qv_sat_l, qv_sat_i, mu, dv, sc, dqsdt, dqsidt, & - ab, abi, kap, eii) - end subroutine get_time_space_phys_variables_c - - subroutine ice_cldliq_collection_c(rho, temp, rhofaci, table_val_qc2qi_collect, qi_incld, qc_incld, ni_incld, & - nc_incld, qc2qi_collect_tend, nc_collect_tend, qc2qr_ice_shed_tend, ncshdc) bind(C) - use micro_p3, only: ice_cldliq_collection - - ! arguments: - real(kind=c_real), value, intent(in) :: rho, temp, rhofaci, table_val_qc2qi_collect - real(kind=c_real), value, intent(in) :: qi_incld, qc_incld, ni_incld, nc_incld - real(kind=c_real), intent(out) :: qc2qi_collect_tend, nc_collect_tend, qc2qr_ice_shed_tend, ncshdc - - call ice_cldliq_collection(rho, temp, rhofaci, table_val_qc2qi_collect, qi_incld, qc_incld, ni_incld, & - nc_incld, qc2qi_collect_tend, nc_collect_tend, qc2qr_ice_shed_tend, ncshdc) - end subroutine ice_cldliq_collection_c - - subroutine ice_rain_collection_c(rho, temp, rhofaci, logn0r, table_val_nr_collect, table_val_qr2qi_collect, & - qi_incld, ni_incld, qr_incld, qr2qi_collect_tend, nr_collect_tend) bind(C) - use micro_p3, only: ice_rain_collection - - ! arguments: - real(kind=c_real), value, intent(in) :: rho, temp, rhofaci, logn0r, table_val_nr_collect, table_val_qr2qi_collect - real(kind=c_real), value, intent(in) :: qi_incld, ni_incld, qr_incld - real(kind=c_real), intent(out) :: qr2qi_collect_tend, nr_collect_tend - - call ice_rain_collection(rho, temp, rhofaci, logn0r, table_val_nr_collect, table_val_qr2qi_collect, & - qi_incld, ni_incld, qr_incld, qr2qi_collect_tend, nr_collect_tend) - end subroutine ice_rain_collection_c - - subroutine ice_self_collection_c(rho, rhofaci, table_val_ni_self_collect, eii, qm_incld, & - qi_incld, ni_incld, ni_selfcollect_tend) bind(C) - use micro_p3, only: ice_self_collection - - ! arguments: - real(kind=c_real), value, intent(in) :: rho, rhofaci, table_val_ni_self_collect, eii, qm_incld - real(kind=c_real), value, intent(in) :: qi_incld, ni_incld - real(kind=c_real), intent(out) :: ni_selfcollect_tend - - call ice_self_collection(rho, rhofaci, table_val_ni_self_collect, eii, qm_incld, & - qi_incld, ni_incld, ni_selfcollect_tend) - end subroutine ice_self_collection_c - - subroutine evaporate_rain_c(qr_incld,qc_incld,nr_incld,qi_incld, & - cld_frac_l,cld_frac_r,qv,qv_prev,qv_sat_l,qv_sat_i, & - ab,abi,epsr,epsi_tot,t,t_prev,latent_heat_sublim,dqsdt,dt,& - qr2qv_evap_tend,nr_evap_tend) bind(C) - use micro_p3, only: evaporate_rain - - ! arguments - real(kind=c_real), value, intent(in) :: qr_incld,qc_incld,nr_incld,qi_incld, & - cld_frac_l,cld_frac_r,qv,qv_prev,qv_sat_l,qv_sat_i, & - ab,abi,epsr,epsi_tot,t,t_prev,latent_heat_sublim,dqsdt,dt - - real(kind=c_real), intent(out) :: qr2qv_evap_tend, nr_evap_tend - - call evaporate_rain(qr_incld,qc_incld,nr_incld,qi_incld, & - cld_frac_l,cld_frac_r,qv,qv_prev,qv_sat_l,qv_sat_i, & - ab,abi,epsr,epsi_tot,t,t_prev,latent_heat_sublim,dqsdt,dt,& - qr2qv_evap_tend,nr_evap_tend) - end subroutine evaporate_rain_c - - subroutine update_prognostic_liquid_c(qc2qr_accret_tend, nc_accret_tend, qc2qr_autoconv_tend,nc2nr_autoconv_tend, ncautr, nc_selfcollect_tend, & - qr2qv_evap_tend, nr_evap_tend, nr_selfcollect_tend, do_predict_nc, do_prescribed_CCN, inv_rho, inv_exner, latent_heat_vapor, dt, th_atm, qv, qc, nc, qr, nr) bind(C) - use micro_p3, only: update_prognostic_liquid - - ! arguments - real(kind=c_real), value, intent(in) :: qc2qr_accret_tend, nc_accret_tend, qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr, nc_selfcollect_tend, & - qr2qv_evap_tend, nr_evap_tend, nr_selfcollect_tend - - logical(kind=c_bool), value, intent(in) :: do_predict_nc - logical(kind=c_bool), value, intent(in) :: do_prescribed_CCN - - real(kind=c_real), value, intent(in) :: inv_rho, inv_exner, latent_heat_vapor, dt - - real(kind=c_real), intent(inout) :: th_atm, qv, qc, nc, qr, nr - - call update_prognostic_liquid(qc2qr_accret_tend, nc_accret_tend, qc2qr_autoconv_tend,nc2nr_autoconv_tend, ncautr, nc_selfcollect_tend, & - qr2qv_evap_tend, nr_evap_tend, nr_selfcollect_tend, do_predict_nc, do_prescribed_CCN, inv_rho, inv_exner, latent_heat_vapor, dt, th_atm, qv, qc, nc, qr, nr) - - end subroutine update_prognostic_liquid_c - - subroutine ice_deposition_sublimation_c(qi_incld, ni_incld, t_atm, qv_sat_l, qv_sat_i, epsi, abi, qv, inv_dt, qidep, qi2qv_sublim_tend, ni_sublim_tend, qiberg) bind(C) - use micro_p3, only : ice_deposition_sublimation - - real(kind=c_real) , value, intent(in) :: qi_incld, ni_incld, t_atm, qv_sat_l, qv_sat_i, epsi, abi, qv, inv_dt - real(kind=c_real) , intent(out) :: qidep, qi2qv_sublim_tend, ni_sublim_tend, qiberg - - call ice_deposition_sublimation(qi_incld, ni_incld, t_atm, qv_sat_l, qv_sat_i, epsi, abi, qv, inv_dt, qidep, qi2qv_sublim_tend, ni_sublim_tend, qiberg) - end subroutine ice_deposition_sublimation_c - - subroutine ice_relaxation_timescale_c(rho, temp, rhofaci, table_val_qi2qr_melting, table_val_qi2qr_vent_melt, & - dv, mu, sc, qi_incld, ni_incld, & - epsi, epsi_tot) bind(C) - use micro_p3, only: calc_ice_relaxation_timescale - - ! arguments - real(kind=c_real), value, intent(in) :: rho, temp, rhofaci, table_val_qi2qr_melting, table_val_qi2qr_vent_melt, & - dv, mu, sc, qi_incld, ni_incld - real(kind=c_real), intent(out) :: epsi - real(kind=c_real), intent(inout) :: epsi_tot - - call calc_ice_relaxation_timescale(rho, temp, rhofaci, table_val_qi2qr_melting, table_val_qi2qr_vent_melt, & - dv, mu, sc, qi_incld, ni_incld, & - epsi, epsi_tot) - end subroutine ice_relaxation_timescale_c - - subroutine calc_liq_relaxation_timescale_c(rho, f1r, f2r, dv, mu, sc, mu_r, & - lamr, cdistr, cdist, qr_incld, & - qc_incld, epsr, epsc) bind(C) - use micro_p3, only: calc_liq_relaxation_timescale - - ! arguments - real(kind=c_real), value, intent(in) :: rho,f1r,f2r,dv,mu,sc,mu_r,lamr, & - cdistr,cdist,qr_incld,qc_incld - real(kind=c_real), intent(out) :: epsr - real(kind=c_real), intent(out) :: epsc - - call calc_liq_relaxation_timescale(rho,f1r,f2r,dv,mu,sc,mu_r,lamr, & - cdistr,cdist,qr_incld,qc_incld,epsr, & - epsc) - end subroutine calc_liq_relaxation_timescale_c - - subroutine ice_nucleation_c(temp, inv_rho, ni, ni_activated, qv_supersat_i, inv_dt, & - do_predict_nc, do_prescribed_CCN, qv2qi_nucleat_tend, ni_nucleat_tend) bind(C) - use micro_p3, only: ice_nucleation - - ! arguments - real(kind=c_real), value, intent(in) :: temp, inv_rho, ni, ni_activated, qv_supersat_i, inv_dt - logical(c_bool), value, intent(in) :: do_predict_nc - logical(c_bool), value, intent(in) :: do_prescribed_CCN - - real(kind=c_real), intent(inout) :: qv2qi_nucleat_tend, ni_nucleat_tend - - call ice_nucleation(temp, inv_rho, ni, ni_activated, qv_supersat_i, inv_dt, & - do_predict_nc, do_prescribed_CCN, qv2qi_nucleat_tend, ni_nucleat_tend) - end subroutine ice_nucleation_c - - subroutine ice_melting_c(rho,T_atm,pres,rhofaci,table_val_qi2qr_melting,table_val_qi2qr_vent_melt,latent_heat_vapor,latent_heat_fusion, & - dv,sc,mu,kap,qv,qi_incld,ni_incld,qi2qr_melt_tend,ni2nr_melt_tend) bind(C) - use micro_p3, only: ice_melting - - ! arguments: - real(kind=c_real), value, intent(in) :: rho,T_atm,pres,rhofaci,table_val_qi2qr_melting,table_val_qi2qr_vent_melt,latent_heat_vapor,latent_heat_fusion,dv,sc,mu,kap,qv,qi_incld,ni_incld - real(kind=c_real), intent(out) :: qi2qr_melt_tend,ni2nr_melt_tend - - call ice_melting(rho,T_atm,pres,rhofaci,table_val_qi2qr_melting,table_val_qi2qr_vent_melt,latent_heat_vapor,latent_heat_fusion,dv,sc,mu,kap,qv,qi_incld,ni_incld,qi2qr_melt_tend,ni2nr_melt_tend) - - end subroutine ice_melting_c - - subroutine ice_cldliq_wet_growth_c(rho, temp, pres, rhofaci, table_val_qi2qr_melting, & - table_val_qi2qr_vent_melt, latent_heat_vapor, latent_heat_fusion, dv, kap, mu, sc, qv, qc_incld, & - qi_incld, ni_incld, qr_incld, & - log_wetgrowth, qr2qi_collect_tend, qc2qi_collect_tend, qc_growth_rate, nr_ice_shed_tend, qc2qr_ice_shed_tend) bind(C) - use micro_p3, only: ice_cldliq_wet_growth - - ! argmens - real(kind=c_real), value, intent(in) :: rho, temp ,pres, rhofaci, table_val_qi2qr_melting, table_val_qi2qr_vent_melt, latent_heat_vapor, latent_heat_fusion, dv, & - kap, mu, sc, qv, qc_incld, qi_incld, ni_incld,qr_incld - logical(kind=c_bool), intent(inout) :: log_wetgrowth - real(kind=c_real), intent(inout) :: qr2qi_collect_tend, qc2qi_collect_tend, qc_growth_rate, nr_ice_shed_tend, qc2qr_ice_shed_tend - - call ice_cldliq_wet_growth(rho, temp, pres, rhofaci, table_val_qi2qr_melting, & - table_val_qi2qr_vent_melt, latent_heat_vapor, latent_heat_fusion, dv, kap, mu, sc, qv, qc_incld, & - qi_incld, ni_incld, qr_incld, & - log_wetgrowth, qr2qi_collect_tend, qc2qi_collect_tend, qc_growth_rate, nr_ice_shed_tend, qc2qr_ice_shed_tend) - end subroutine ice_cldliq_wet_growth_c - - subroutine get_latent_heat_c(its,ite,kts,kte,v,s,f) bind(C) - use micro_p3, only: get_latent_heat - - ! arguments - integer(kind=c_int), intent(in), value :: its, ite, kts, kte - real(kind=c_real), dimension(its:ite, kts:kte), intent(out) :: v, s, f - - call get_latent_heat(its,ite,kts,kte,v,s,f) - end subroutine get_latent_heat_c - - function subgrid_variance_scaling_c(relvar,expon) result(res) bind(C) - use micro_p3, only: subgrid_variance_scaling - - ! arguments - real(kind=c_real), value, intent(in) :: relvar,expon - real(kind=c_real) :: res - - res = subgrid_variance_scaling(relvar,expon) - return - end function subgrid_variance_scaling_c - - subroutine check_values_c(qv, temp, kts, kte, timestepcount, & - force_abort, source_ind, col_loc) bind(C) - use micro_p3, only: check_values - - ! argmens - real(kind=c_real), intent(in) :: qv(kts:kte), temp(kts:kte), col_loc(3) - integer(kind=c_int), value, intent(in) :: kts, kte, timestepcount, source_ind - logical(kind=c_bool), value, intent(in) :: force_abort - - call check_values(qv,Temp,kts,kte,timestepcount,force_abort,source_ind,col_loc) - end subroutine check_values_c - - subroutine calculate_incloud_mixingratios_c(qc, qr, qi, qm, nc, nr, ni, bm, & - inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, & - qc_incld, qr_incld, qi_incld, qm_incld, & - nc_incld, nr_incld, ni_incld, bm_incld) bind(C) - use micro_p3, only: calculate_incloud_mixingratios - - ! argumens - real(kind=c_real), value, intent(in) :: qc, qr, qi, qm, nc, nr, ni, bm, inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r - real(kind=c_real), intent(inout) :: qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, ni_incld, bm_incld - - call calculate_incloud_mixingratios(qc, qr, qi, qm, nc, nr, ni, bm, & - inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, & - qc_incld, qr_incld, qi_incld, qm_incld, & - nc_incld, nr_incld, ni_incld, bm_incld) - end subroutine calculate_incloud_mixingratios_c - - subroutine p3_main_part1_c(kts, kte, kbot, ktop, kdir, do_predict_nc, do_prescribed_CCN, dt, & - pres, dpres, dz, nc_nuceat_tend, inv_exner, exner, inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, nccn_prescribed, & - T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, qv, th_atm, qc, nc, qr, nr, & - qi, ni, qm, bm, qc_incld, qr_incld, qi_incld, qm_incld, & - nc_incld, nr_incld, ni_incld, bm_incld, is_nucleat_possible, is_hydromet_present) bind(C) - - use micro_p3, only: p3_main_part1 - - ! arguments - integer(kind=c_int), value, intent(in) :: kts, kte, kbot, ktop, kdir - logical(kind=c_bool), value, intent(in) :: do_predict_nc, do_prescribed_CCN - real(kind=c_real), value, intent(in) :: dt - - real(kind=c_real), intent(in), dimension(kts:kte) :: pres, dpres, dz, nc_nuceat_tend, inv_exner, exner, inv_cld_frac_l, inv_cld_frac_i, & - inv_cld_frac_r, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, nccn_prescribed - - real(kind=c_real), intent(inout), dimension(kts:kte) :: T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, & - acn, qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, qc_incld, qr_incld, qi_incld, & - qm_incld, nc_incld, nr_incld, ni_incld, bm_incld - - logical(kind=c_bool), intent(out) :: is_nucleat_possible, is_hydromet_present - - call p3_main_part1(kts, kte, kbot, ktop, kdir, do_predict_nc, do_prescribed_CCN, dt, & - pres, dpres, dz, nc_nuceat_tend, inv_exner, exner, inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, nccn_prescribed, & - T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, qv, th_atm, qc, nc, qr, nr, & - qi, ni, qm, bm, qc_incld, qr_incld, qi_incld, qm_incld, & - nc_incld, nr_incld, ni_incld, bm_incld, is_nucleat_possible, is_hydromet_present) - - end subroutine p3_main_part1_c - - subroutine p3_main_part2_c(kts, kte, kbot, ktop, kdir, do_predict_nc, do_prescribed_CCN, dt, inv_dt, & - pres, inv_exner, inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, & - ni_activated, inv_qc_relvar, cld_frac_i, cld_frac_l, cld_frac_r, qv_prev, t_prev, & - T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofaci, acn, qv, th_atm, qc, nc, qr, nr, qi, ni, & - qm, bm, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, & - ni_incld, bm_incld, mu_c, nu, lamc, cdist, cdist1, cdistr, mu_r, lamr, logn0r, qv2qi_depos_tend, precip_total_tend, & - nevapr, qr_evap_tend, vap_liq_exchange, vap_ice_exchange, liq_ice_exchange, pratot, & - prctot, is_hydromet_present) bind(C) - - use micro_p3, only: p3_main_part2 - - !arguments - integer(kind=c_int), value, intent(in) :: kts, kte, kbot, ktop, kdir - logical(kind=c_bool), value, intent(in) :: do_predict_nc, do_prescribed_CCN - real(kind=c_real), value, intent(in) :: dt, inv_dt - - real(kind=c_real), intent(in), dimension(kts:kte) :: pres, inv_exner, inv_cld_frac_l, inv_cld_frac_i, & - inv_cld_frac_r, ni_activated, inv_qc_relvar, cld_frac_i, cld_frac_l, cld_frac_r, qv_prev, t_prev - - real(kind=c_real), intent(inout), dimension(kts:kte) :: T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofaci, acn, & - qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, qc_incld, qr_incld, & - qi_incld, qm_incld, nc_incld, nr_incld, ni_incld, bm_incld, mu_c, nu, lamc, cdist, cdist1, & - cdistr, mu_r, lamr, logn0r, qv2qi_depos_tend, precip_total_tend, nevapr, qr_evap_tend, vap_liq_exchange, & - vap_ice_exchange, liq_ice_exchange, pratot, prctot - - logical(kind=c_bool), intent(out) :: is_hydromet_present - - ! throwaway - real(kind=c_real), dimension(kts:kte,49) :: p3_tend_out - - call p3_main_part2(kts, kte, kbot, ktop, kdir, do_predict_nc, do_prescribed_CCN, dt, inv_dt, & - pres, inv_exner, inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, ni_activated, inv_qc_relvar, cld_frac_i, cld_frac_l, cld_frac_r, qv_prev, t_prev, & - T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofaci, acn, qv, th_atm, qc, nc, qr, nr, qi, ni, & - qm, bm, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, & - ni_incld, bm_incld, mu_c, nu, lamc, cdist, cdist1, cdistr, mu_r, lamr, logn0r, qv2qi_depos_tend, precip_total_tend, & - nevapr, qr_evap_tend, vap_liq_exchange, vap_ice_exchange, liq_ice_exchange, pratot, & - prctot, p3_tend_out, is_hydromet_present) - - end subroutine p3_main_part2_c - - subroutine p3_main_part3_c(kts, kte, kbot, ktop, kdir, & - inv_exner, cld_frac_l, cld_frac_r, cld_frac_i, & - rho, inv_rho, rhofaci, qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, & - mu_c, nu, lamc, mu_r, lamr, vap_liq_exchange, & - ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, rho_qi, diag_equiv_reflectivity, diag_eff_radius_qc, diag_eff_radius_qr) bind(C) - - use micro_p3, only: p3_main_part3 - - ! args - - integer(kind=c_int), value, intent(in) :: kts, kte, kbot, ktop, kdir - real(kind=c_real), intent(in), dimension(kts:kte) :: inv_exner, cld_frac_l, cld_frac_r, cld_frac_i - real(kind=c_real), intent(inout), dimension(kts:kte) :: rho, inv_rho, rhofaci, & - qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, & - mu_c, nu, lamc, mu_r, & - lamr, vap_liq_exchange, & - ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, rho_qi, & - diag_equiv_reflectivity, diag_eff_radius_qc, diag_eff_radius_qr - - call p3_main_part3(kts, kte, kbot, ktop, kdir, & - inv_exner, cld_frac_l, cld_frac_r, cld_frac_i, & - rho, inv_rho, rhofaci, qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, & - mu_c, nu, lamc, mu_r, lamr, vap_liq_exchange, & - ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, rho_qi, diag_equiv_reflectivity, diag_eff_radius_qc, diag_eff_radius_qr) - - end subroutine p3_main_part3_c - - subroutine ice_supersat_conservation_c(qidep, qinuc, cld_frac_i, qv, qv_sat_i, latent_heat_sublim, t_atm, dt, qi2qv_sublim_tend, qr2qv_evap_tend) bind(C) - use micro_p3, only : ice_supersat_conservation - - real(kind=c_real) , intent(inout) :: qidep, qinuc - real(kind=c_real) , value, intent(in) :: cld_frac_i, qv, qv_sat_i, latent_heat_sublim, t_atm, dt, qi2qv_sublim_tend, qr2qv_evap_tend - - call ice_supersat_conservation(qidep, qinuc, cld_frac_i, qv, qv_sat_i, latent_heat_sublim, t_atm, dt, qi2qv_sublim_tend, qr2qv_evap_tend) - end subroutine ice_supersat_conservation_c - subroutine nc_conservation_c(nc, nc_selfcollect_tend, dt, nc_collect_tend, nc2ni_immers_freeze_tend, nc_accret_tend, nc2nr_autoconv_tend) bind(C) - use micro_p3, only : nc_conservation - - real(kind=c_real) , value, intent(in) :: nc, nc_selfcollect_tend, dt - real(kind=c_real) , intent(inout) :: nc_collect_tend, nc2ni_immers_freeze_tend, nc_accret_tend, nc2nr_autoconv_tend - - call nc_conservation(nc, nc_selfcollect_tend, dt, nc_collect_tend, nc2ni_immers_freeze_tend, nc_accret_tend, nc2nr_autoconv_tend) - end subroutine nc_conservation_c - subroutine nr_conservation_c(nr, ni2nr_melt_tend, nr_ice_shed_tend, ncshdc, nc2nr_autoconv_tend, dt, nmltratio, nr_collect_tend, nr2ni_immers_freeze_tend, nr_selfcollect_tend, nr_evap_tend) bind(C) - use micro_p3, only : nr_conservation - - real(kind=c_real) , value, intent(in) :: nr, ni2nr_melt_tend, nr_ice_shed_tend, ncshdc, nc2nr_autoconv_tend, dt, nmltratio - real(kind=c_real) , intent(inout) :: nr_collect_tend, nr2ni_immers_freeze_tend, nr_selfcollect_tend, nr_evap_tend - - call nr_conservation(nr, ni2nr_melt_tend, nr_ice_shed_tend, ncshdc, nc2nr_autoconv_tend, dt, nmltratio, nr_collect_tend, nr2ni_immers_freeze_tend, nr_selfcollect_tend, nr_evap_tend) - end subroutine nr_conservation_c - subroutine ni_conservation_c(ni, ni_nucleat_tend, nr2ni_immers_freeze_tend, nc2ni_immers_freeze_tend, dt, ni2nr_melt_tend, ni_sublim_tend, ni_selfcollect_tend) bind(C) - use micro_p3, only : ni_conservation - - real(kind=c_real) , value, intent(in) :: ni, ni_nucleat_tend, nr2ni_immers_freeze_tend, nc2ni_immers_freeze_tend, dt - real(kind=c_real) , intent(inout) :: ni2nr_melt_tend, ni_sublim_tend, ni_selfcollect_tend - - call ni_conservation(ni, ni_nucleat_tend, nr2ni_immers_freeze_tend, nc2ni_immers_freeze_tend, dt, ni2nr_melt_tend, ni_sublim_tend, ni_selfcollect_tend) - end subroutine ni_conservation_c - subroutine prevent_liq_supersaturation_c(pres, t_atm, qv, latent_heat_vapor, latent_heat_sublim, dt, qidep, qinuc, qi2qv_sublim_tend, qr2qv_evap_tend) bind(C) - use micro_p3, only : prevent_liq_supersaturation - - real(kind=c_real) , value, intent(in) :: pres, t_atm, qv, latent_heat_vapor, latent_heat_sublim, dt, qidep, qinuc - real(kind=c_real) , intent(inout) :: qi2qv_sublim_tend, qr2qv_evap_tend - - call prevent_liq_supersaturation(pres, t_atm, qv, latent_heat_vapor, latent_heat_sublim, dt, qidep, qinuc, qi2qv_sublim_tend, qr2qv_evap_tend) - end subroutine prevent_liq_supersaturation_c -end module p3_iso_c diff --git a/components/eamxx/src/physics/p3/p3_main_wrap.cpp b/components/eamxx/src/physics/p3/p3_main_wrap.cpp deleted file mode 100644 index ed46e281a675..000000000000 --- a/components/eamxx/src/physics/p3/p3_main_wrap.cpp +++ /dev/null @@ -1,76 +0,0 @@ -#include "p3_main_wrap.hpp" -#include "p3_f90.hpp" -#include "p3_functions_f90.hpp" -#include "physics_constants.hpp" -#include "p3_ic_cases.hpp" - -#include "ekat/ekat_assert.hpp" - -using scream::Real; -using scream::Int; -extern "C" { - void p3_main_c(Real* qc, Real* nc, Real* qr, Real* nr, Real* th_atm, - Real* qv, Real dt, Real* qi, Real* qm, - Real* ni, Real* bm, Real* pres, - Real* dz, Real* nc_nuceat_tend, Real* nccn_prescribed, Real* ni_activated, Real* inv_qc_relvar, - Int it, Real* precip_liq_surf, Real* precip_ice_surf, Int its, - Int ite, Int kts, Int kte, Real* diag_eff_radius_qc, Real* diag_eff_radius_qi, Real* diag_eff_radius_qr, - Real* rho_qi, bool do_predict_nc, bool do_prescribed_CCN, Real* dpres, Real* inv_exner, - Real* qv2qi_depos_tend, - Real* precip_liq_flux, Real* precip_ice_flux, // 1 extra column size - Real* cld_frac_r, Real* cld_frac_l, Real* cld_frac_i, - Real* liq_ice_exchange, Real* vap_liq_exchange, - Real* vap_ice_exchange, Real* qv_prev, Real* t_prev, Real* elapsed_s); -} - -namespace scream { -namespace p3 { - -Int p3_main_wrap(const FortranData& d, bool use_fortran) { - EKAT_REQUIRE_MSG(d.dt > 0, "invalid dt"); - if (use_fortran) { - Real elapsed_s; - p3_main_c(d.qc.data(), d.nc.data(), d.qr.data(), d.nr.data(), - d.th_atm.data(), d.qv.data(), d.dt, d.qi.data(), - d.qm.data(), d.ni.data(), d.bm.data(), - d.pres.data(), d.dz.data(), d.nc_nuceat_tend.data(), d.nccn_prescribed.data(), d.ni_activated.data(), d.inv_qc_relvar.data(), - d.it, d.precip_liq_surf.data(), d.precip_ice_surf.data(), 1, d.ncol, 1, d.nlev, - d.diag_eff_radius_qc.data(), d.diag_eff_radius_qi.data(), d.diag_eff_radius_qr.data(), d.rho_qi.data(), - d.do_predict_nc, d.do_prescribed_CCN, d.dpres.data(), d.inv_exner.data(), d.qv2qi_depos_tend.data(), - d.precip_liq_flux.data(), d.precip_ice_flux.data(), d.cld_frac_r.data(), d.cld_frac_l.data(), d.cld_frac_i.data(), - d.liq_ice_exchange.data(), d.vap_liq_exchange.data(),d.vap_ice_exchange.data(),d.qv_prev.data(),d.t_prev.data(), &elapsed_s); - return static_cast(elapsed_s * 1000000); - } - else { - return p3_main_f(d.qc.data(), d.nc.data(), d.qr.data(), d.nr.data(), d.th_atm.data(), - d.qv.data(), d.dt, d.qi.data(), d.qm.data(), d.ni.data(), - d.bm.data(), d.pres.data(), d.dz.data(), d.nc_nuceat_tend.data(), d.nccn_prescribed.data(), - d.ni_activated.data(), d.inv_qc_relvar.data(), d.it, d.precip_liq_surf.data(), - d.precip_ice_surf.data(), 1, d.ncol, 1, d.nlev, d.diag_eff_radius_qc.data(), - d.diag_eff_radius_qi.data(), d.diag_eff_radius_qr.data(), d.rho_qi.data(), d.do_predict_nc, d.do_prescribed_CCN, - d.dpres.data(), d.inv_exner.data(), d.qv2qi_depos_tend.data(), - d.precip_liq_flux.data(), d.precip_ice_flux.data(), - d.cld_frac_r.data(), d.cld_frac_l.data(), d.cld_frac_i.data(), - d.liq_ice_exchange.data(), d.vap_liq_exchange.data(), - d.vap_ice_exchange.data(),d.qv_prev.data(),d.t_prev.data() ); - - } -} - -int test_p3_init () { - p3_init(); - P3GlobalForFortran::deinit(); - return 0; -} - -int test_p3_ic (bool use_fortran) { - const auto d = ic::Factory::create(ic::Factory::mixed); - d->dt = 300.0; - p3_init(); - p3_main_wrap(*d, use_fortran); - P3GlobalForFortran::deinit(); - return 0; -} - -} // namespace p3 -} // namespace scream diff --git a/components/eamxx/src/physics/p3/p3_tables_setup.cpp b/components/eamxx/src/physics/p3/p3_tables_setup.cpp deleted file mode 100644 index 8582674d1fe5..000000000000 --- a/components/eamxx/src/physics/p3/p3_tables_setup.cpp +++ /dev/null @@ -1,8 +0,0 @@ -// This is a tiny program that calls p3_init() to generate tables used by p3 - -#include "physics/p3/p3_f90.hpp" - -int main(int /* argc */, char** /* argv */) { - scream::p3::p3_init(/* write_tables = */ true); - return 0; -} diff --git a/components/eamxx/src/physics/p3/tests/CMakeLists.txt b/components/eamxx/src/physics/p3/tests/CMakeLists.txt index 217c2945e489..b8d6c7d2b7a4 100644 --- a/components/eamxx/src/physics/p3/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/tests/CMakeLists.txt @@ -1,5 +1,7 @@ include(ScreamUtils) +add_subdirectory(infra) + set(P3_TESTS_SRCS p3_tests.cpp p3_unit_tests.cpp @@ -44,75 +46,73 @@ else () set (FORCE_RUN_DIFF_FAILS "") endif() -# NOTE: tests inside this if statement won't be built in a baselines-only build -if (NOT SCREAM_ONLY_GENERATE_BASELINES) - CreateUnitTest(p3_tests "${P3_TESTS_SRCS}" - LIBS p3 - THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} - LABELS "p3;physics") - - # Make sure that a diff in the two implementation triggers a failed test (in debug only) - CreateUnitTest (p3_tests_fail p3_rain_sed_unit_tests.cpp - LIBS p3 - COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF - THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} - LABELS "p3;physics;fail" - ${FORCE_RUN_DIFF_FAILS}) - - if (NOT SCREAM_P3_SMALL_KERNELS) - CreateUnitTest(p3_sk_tests "${P3_TESTS_SRCS}" - LIBS p3_sk - THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} - LABELS "p3_sk;physics") - - # Make sure that a diff in the two implementation triggers a failed test (in debug only) - CreateUnitTest (p3_sk_tests_fail p3_rain_sed_unit_tests.cpp - LIBS p3_sk - COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF - THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} - LABELS "p3_sk;physics;fail" - ${FORCE_RUN_DIFF_FAILS}) - endif() -endif() - +# All tests should understand the same baseline args if (SCREAM_ENABLE_BASELINE_TESTS) if (SCREAM_ONLY_GENERATE_BASELINES) - set(BASELINE_FILE_ARG "-g -b ${SCREAM_BASELINES_DIR}/data/p3_run_and_cmp.baseline") + set(BASELINE_FILE_ARG "-g -b ${SCREAM_BASELINES_DIR}/data") + # We don't want to do thread spreads when generating. That + # could cause race conditions in the file system. + set(P3_THREADS "${SCREAM_TEST_MAX_THREADS}") else() - set(BASELINE_FILE_ARG "-b ${SCREAM_BASELINES_DIR}/data/p3_run_and_cmp.baseline") + set(BASELINE_FILE_ARG "-c -b ${SCREAM_BASELINES_DIR}/data") + set(P3_THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC}) endif() +else() + set(BASELINE_FILE_ARG "-n") # no baselines + set(P3_THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC}) +endif() + +CreateUnitTest(p3_tests "${P3_TESTS_SRCS}" + LIBS p3 p3_test_infra + EXE_ARGS "--args ${BASELINE_FILE_ARG}" + THREADS ${P3_THREADS} + LABELS "p3;physics;baseline_gen;baseline_cmp") - CreateUnitTestExec(p3_run_and_cmp "p3_run_and_cmp.cpp" - LIBS p3 - EXCLUDE_MAIN_CPP) +# Make sure that a diff in the two implementation triggers a failed test (in debug only) +# No need to run lots of different thread counts. +if (SCREAM_ENABLE_BASELINE_TESTS) + CreateUnitTest (p3_tests_fail p3_rain_sed_unit_tests.cpp + LIBS p3 p3_test_infra + EXE_ARGS "--args ${BASELINE_FILE_ARG}" + COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF + LABELS "p3;physics;fail" + ${FORCE_RUN_DIFF_FAILS}) +endif() - CreateUnitTestFromExec(p3_run_and_cmp_cxx p3_run_and_cmp - THREADS ${SCREAM_TEST_MAX_THREADS} - EXE_ARGS "${BASELINE_FILE_ARG}" - LABELS "p3;physics") +# If small kernels are ON, we don't need a separate executable to test them. +# Also, we never want to generate baselines with this separate executable +if (NOT SCREAM_P3_SMALL_KERNELS AND NOT SCREAM_ONLY_GENERATE_BASELINES) + # Note: Only the p3_main test does something different when + # small kernels are on. The SK dispatch routines are mostly trivial + # and it's not worth adding tons of test infrastructure to support + # BFB unit tests for these. + CreateUnitTest(p3_sk_tests "p3_main_unit_tests.cpp" + LIBS p3_sk p3_test_infra + EXE_ARGS "--args ${BASELINE_FILE_ARG}" + THREADS ${P3_THREADS} + LABELS "p3_sk;physics;baseline_cmp") +endif() - CreateUnitTestFromExec(p3_run_and_cmp_f90 p3_run_and_cmp - THREADS ${SCREAM_TEST_MAX_THREADS} - EXE_ARGS "-f ${BASELINE_FILE_ARG}" - LABELS "p3;physics") +# Note: the baseline_gen label label is really only used if SCREAM_ONLY_GENERATE_BASELINES=ON, but no harm adding it +CreateUnitTest(p3_run_and_cmp "p3_run_and_cmp.cpp" + LIBS p3 p3_test_infra + EXCLUDE_MAIN_CPP + THREADS ${SCREAM_TEST_MAX_THREADS} + EXE_ARGS "${BASELINE_FILE_ARG}" + LABELS "p3;physics;baseline_gen;baseline_cmp") - # Make sure that a diff from baselines triggers a failed test (in debug only) - CreateUnitTest(p3_run_and_cmp_cxx_fail "p3_run_and_cmp.cpp" - LIBS p3 - COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF - THREADS ${SCREAM_TEST_MAX_THREADS} - EXE_ARGS "${BASELINE_FILE_ARG}" - LABELS "p3;physics;fail" - EXCLUDE_MAIN_CPP - ${FORCE_RUN_DIFF_FAILS}) +# This executable can be used to re-generate tables in ${SCREAM_DATA_DIR} +add_executable(p3_tables_setup EXCLUDE_FROM_ALL p3_tables_setup.cpp) +target_link_libraries(p3_tables_setup p3) - # By default, baselines should be created using all fortran (ctest -L baseline_gen). If the user wants - # to use CXX to generate their baselines, they should use "ctest -L baseline_gen_cxx". - # Note: the baseline_gen label label is really only used if SCREAM_ONLY_GENERATE_BASELINES=ON, but no harm adding it - if (SCREAM_TEST_MAX_THREADS GREATER 1) - # ECUT only adds _ompX if we have more than one value of X, or if X>1 - set (TEST_SUFFIX _omp${SCREAM_TEST_MAX_THREADS}) - endif() - set_tests_properties (p3_run_and_cmp_f90${TEST_SUFFIX} PROPERTIES LABELS "baseline_gen;baseline_cmp") - set_tests_properties (p3_run_and_cmp_cxx${TEST_SUFFIX} PROPERTIES LABELS "baseline_gen;cxx baseline_cmp") +# Make sure that a diff from baselines triggers a failed test (in debug only) +if (SCREAM_ENABLE_BASELINE_TESTS) + CreateUnitTest(p3_run_and_cmp_fail "p3_run_and_cmp.cpp" + LIBS p3 p3_test_infra + COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF + THREADS ${SCREAM_TEST_MAX_THREADS} + EXE_ARGS "${BASELINE_FILE_ARG}" + LABELS "p3;physics;fail" + EXCLUDE_MAIN_CPP + ${FORCE_RUN_DIFF_FAILS}) endif() diff --git a/components/eamxx/src/physics/p3/tests/infra/CMakeLists.txt b/components/eamxx/src/physics/p3/tests/infra/CMakeLists.txt new file mode 100644 index 000000000000..6f7093a33ec7 --- /dev/null +++ b/components/eamxx/src/physics/p3/tests/infra/CMakeLists.txt @@ -0,0 +1,15 @@ +set(INFRA_SRCS + p3_data.cpp + p3_ic_cases.cpp + p3_main_wrap.cpp + p3_test_data.cpp +) + +#crusher change +if (Kokkos_ENABLE_HIP) +set_source_files_properties(p3_test_data.cpp PROPERTIES COMPILE_FLAGS -O0) +endif() + +add_library(p3_test_infra ${INFRA_SRCS}) +target_link_libraries(p3_test_infra p3) +target_include_directories(p3_test_infra PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/components/eamxx/src/physics/p3/p3_f90.cpp b/components/eamxx/src/physics/p3/tests/infra/p3_data.cpp similarity index 75% rename from components/eamxx/src/physics/p3/p3_f90.cpp rename to components/eamxx/src/physics/p3/tests/infra/p3_data.cpp index 38bb84c416ec..24fd6529f22d 100644 --- a/components/eamxx/src/physics/p3/p3_f90.cpp +++ b/components/eamxx/src/physics/p3/tests/infra/p3_data.cpp @@ -1,4 +1,4 @@ -#include "p3_f90.hpp" +#include "p3_data.hpp" #include "physics_constants.hpp" #include "p3_ic_cases.hpp" @@ -6,17 +6,11 @@ using scream::Real; using scream::Int; -extern "C" { - void micro_p3_utils_init_c(Real Cpair, Real Rair, Real RH2O, Real RHO_H2O, - Real MWH2O, Real MWdry, Real gravit, Real LatVap, Real LatIce, - Real CpLiq, Real Tmelt, Real Pi, bool masterproc); - void p3_init_c(const char** lookup_file_dir, int* info, const bool& write_tables); -} namespace scream { namespace p3 { -FortranData::FortranData (Int ncol_, Int nlev_) +P3Data::P3Data (Int ncol_, Int nlev_) : ncol(ncol_), nlev(nlev_) { do_predict_nc = true; @@ -62,11 +56,11 @@ FortranData::FortranData (Int ncol_, Int nlev_) vap_ice_exchange = Array2("sum of vap-ice phase change tendenices", ncol, nlev); } -FortranDataIterator::FortranDataIterator (const FortranData::Ptr& d) { +P3DataIterator::P3DataIterator (const P3Data::Ptr& d) { init(d); } -void FortranDataIterator::init (const FortranData::Ptr& dp) { +void P3DataIterator::init (const P3Data::Ptr& dp) { d_ = dp; #define fdipb(name) \ fields_.push_back({#name, \ @@ -79,7 +73,7 @@ void FortranDataIterator::init (const FortranData::Ptr& dp) { fdipb(nc); fdipb(qr); fdipb(nr); fdipb(qi); fdipb(ni); fdipb(qm); fdipb(bm); fdipb(precip_liq_surf); fdipb(precip_ice_surf); fdipb(diag_eff_radius_qc); fdipb(diag_eff_radius_qi); fdipb(diag_eff_radius_qr); fdipb(rho_qi); - fdipb(dpres); fdipb(inv_exner); fdipb(qv2qi_depos_tend); + fdipb(dpres); fdipb(inv_exner); fdipb(qv2qi_depos_tend); fdipb(precip_liq_flux); fdipb(precip_ice_flux); fdipb(cld_frac_r); fdipb(cld_frac_l); fdipb(cld_frac_i); fdipb(liq_ice_exchange); fdipb(vap_liq_exchange); @@ -87,33 +81,14 @@ void FortranDataIterator::init (const FortranData::Ptr& dp) { #undef fdipb } -const FortranDataIterator::RawArray& -FortranDataIterator::getfield (Int i) const { +const P3DataIterator::RawArray& +P3DataIterator::getfield (Int i) const { EKAT_ASSERT(i >= 0 || i < nfield()); return fields_[i]; } -void micro_p3_utils_init (const bool masterproc) { - using c = scream::physics::Constants; - micro_p3_utils_init_c(c::Cpair, c::Rair, c::RH2O, c::RHO_H2O, - c::MWH2O, c::MWdry, c::gravit, c::LatVap, c::LatIce, - c::CpLiq, c::Tmelt, c::Pi, masterproc); -} - -void p3_init (const bool write_tables, const bool masterproc) { - static bool is_init = false; - if (!is_init) { - micro_p3_utils_init(masterproc); - static const char* dir = SCREAM_DATA_DIR "/tables"; - Int info; - p3_init_c(&dir, &info, write_tables); - EKAT_REQUIRE_MSG(info == 0, "p3_init_c returned info " << info); - is_init = true; - } -} - -int test_FortranData () { - FortranData d(11, 72); +int test_P3Data () { + P3Data d(11, 72); return 0; } diff --git a/components/eamxx/src/physics/p3/p3_f90.hpp b/components/eamxx/src/physics/p3/tests/infra/p3_data.hpp similarity index 60% rename from components/eamxx/src/physics/p3/p3_f90.hpp rename to components/eamxx/src/physics/p3/tests/infra/p3_data.hpp index d07524d9c7a2..55e74300215c 100644 --- a/components/eamxx/src/physics/p3/p3_f90.hpp +++ b/components/eamxx/src/physics/p3/tests/infra/p3_data.hpp @@ -1,5 +1,5 @@ -#ifndef SCREAM_P3_F90_HPP -#define SCREAM_P3_F90_HPP +#ifndef SCREAM_P3_DATA_HPP +#define SCREAM_P3_DATA_HPP #include "share/scream_types.hpp" @@ -9,16 +9,16 @@ namespace scream { namespace p3 { -// Data format we can use to communicate with Fortran version. -struct FortranData { - typedef std::shared_ptr Ptr; +// Data format we can use to store (and read/write) data for a full P3 run. +struct P3Data { + typedef std::shared_ptr Ptr; using KT = KokkosTypes; using Scalar = Real; - using Array1 = typename KT::template lview; - using Array2 = typename KT::template lview; - using Array3 = typename KT::template lview; + using Array1 = typename KT::template view_1d; + using Array2 = typename KT::template view_2d; + using Array3 = typename KT::template view_3d; bool do_predict_nc; bool do_prescribed_CCN; @@ -36,41 +36,38 @@ struct FortranData { Array3 p3_tend_out; Array2 liq_ice_exchange,vap_liq_exchange,vap_ice_exchange; - FortranData(Int ncol, Int nlev); + P3Data(Int ncol, Int nlev); }; -// Iterate over a FortranData's arrays. For examples, see Baseline::write, read. -struct FortranDataIterator { +// Iterate over a P3Data's arrays. For examples, see Baseline::write, read. +struct P3DataIterator { struct RawArray { std::string name; Int dim; Int extent[3]; - FortranData::Scalar* data; - FortranData::Array1::size_type size; + P3Data::Scalar* data; + P3Data::Array1::size_type size; }; - explicit FortranDataIterator(const FortranData::Ptr& d); + explicit P3DataIterator(const P3Data::Ptr& d); Int nfield () const { return fields_.size(); } const RawArray& getfield(Int i) const; private: - FortranData::Ptr d_; + P3Data::Ptr d_; std::vector fields_; - void init(const FortranData::Ptr& d); + void init(const P3Data::Ptr& d); }; -void p3_init(const bool write_tables = false, - const bool masterproc = false); - // We will likely want to remove these checks in the future, as we're not tied // to the exact implementation or arithmetic in P3. For now, these checks are // here to establish that the initial regression-testing code gives results that // match the python f2py tester, without needing a data file. -Int check_against_python(const FortranData& d); +Int check_against_python(const P3Data& d); -int test_FortranData(); +int test_P3Data(); } // namespace p3 } // namespace scream diff --git a/components/eamxx/src/physics/p3/p3_ic_cases.cpp b/components/eamxx/src/physics/p3/tests/infra/p3_ic_cases.cpp similarity index 95% rename from components/eamxx/src/physics/p3/p3_ic_cases.cpp rename to components/eamxx/src/physics/p3/tests/infra/p3_ic_cases.cpp index 1560dc0afb28..52b70de6025b 100644 --- a/components/eamxx/src/physics/p3/p3_ic_cases.cpp +++ b/components/eamxx/src/physics/p3/tests/infra/p3_ic_cases.cpp @@ -8,12 +8,12 @@ namespace p3 { namespace ic { // From mixed_case_data.py in scream-docs at commit 4bbea4. -FortranData::Ptr make_mixed (const Int ncol, const Int nlev) { +P3Data::Ptr make_mixed (const Int ncol, const Int nlev) { using consts = scream::physics::Constants; const Int nk = nlev; Int k; - const auto dp = std::make_shared(ncol, nk); + const auto dp = std::make_shared(ncol, nk); auto& d = *dp; for (Int i = 0; i < ncol; ++i) { @@ -66,7 +66,7 @@ FortranData::Ptr make_mixed (const Int ncol, const Int nlev) { // To get potential temperature, start by making absolute temperature vary // between 150K at top of atmos and 300k at surface, then convert to potential // temp. - FortranData::Array1 T_atm("T", nk); + P3Data::Array1 T_atm("T", nk); for (k = 0; k < nk; ++k) { T_atm(k) = 150 + 150/double(nk)*k; if (i > 0) T_atm(k) += ((i % 3) - 0.5)/double(nk)*k; @@ -119,7 +119,7 @@ FortranData::Ptr make_mixed (const Int ncol, const Int nlev) { return dp; } -FortranData::Ptr Factory::create (IC ic, Int ncol, Int nlev) { +P3Data::Ptr Factory::create (IC ic, Int ncol, Int nlev) { switch (ic) { case mixed: return make_mixed(ncol, nlev); default: diff --git a/components/eamxx/src/physics/p3/p3_ic_cases.hpp b/components/eamxx/src/physics/p3/tests/infra/p3_ic_cases.hpp similarity index 64% rename from components/eamxx/src/physics/p3/p3_ic_cases.hpp rename to components/eamxx/src/physics/p3/tests/infra/p3_ic_cases.hpp index a8c461b06daa..bbf6f133e516 100644 --- a/components/eamxx/src/physics/p3/p3_ic_cases.hpp +++ b/components/eamxx/src/physics/p3/tests/infra/p3_ic_cases.hpp @@ -1,18 +1,18 @@ #ifndef INCLUDE_SCREAM_P3_IC_CASES_HPP #define INCLUDE_SCREAM_P3_IC_CASES_HPP -#include "p3_f90.hpp" +#include "p3_data.hpp" namespace scream { namespace p3 { namespace ic { -FortranData::Ptr make_mixed(Int ncol); +P3Data::Ptr make_mixed(Int ncol); struct Factory { enum IC { mixed }; - static FortranData::Ptr create(IC ic, Int ncol = 1, Int nlev = 72); + static P3Data::Ptr create(IC ic, Int ncol = 1, Int nlev = 72); }; } // namespace ic diff --git a/components/eamxx/src/physics/p3/tests/infra/p3_main_wrap.cpp b/components/eamxx/src/physics/p3/tests/infra/p3_main_wrap.cpp new file mode 100644 index 000000000000..d4faa4c70527 --- /dev/null +++ b/components/eamxx/src/physics/p3/tests/infra/p3_main_wrap.cpp @@ -0,0 +1,49 @@ +#include "p3_main_wrap.hpp" +#include "p3_data.hpp" +#include "p3_test_data.hpp" +#include "physics_constants.hpp" +#include "p3_ic_cases.hpp" + +#include "ekat/ekat_assert.hpp" + +using scream::Real; +using scream::Int; + +namespace scream { +namespace p3 { + +Int p3_main_wrap(const P3Data& d) { + EKAT_REQUIRE_MSG(d.dt > 0, "invalid dt"); + return p3_main_host(d.qc.data(), d.nc.data(), d.qr.data(), d.nr.data(), d.th_atm.data(), + d.qv.data(), d.dt, d.qi.data(), d.qm.data(), d.ni.data(), + d.bm.data(), d.pres.data(), d.dz.data(), d.nc_nuceat_tend.data(), d.nccn_prescribed.data(), + d.ni_activated.data(), d.inv_qc_relvar.data(), d.it, d.precip_liq_surf.data(), + d.precip_ice_surf.data(), 1, d.ncol, 1, d.nlev, d.diag_eff_radius_qc.data(), + d.diag_eff_radius_qi.data(), d.diag_eff_radius_qr.data(), d.rho_qi.data(), d.do_predict_nc, d.do_prescribed_CCN, + d.dpres.data(), d.inv_exner.data(), d.qv2qi_depos_tend.data(), + d.precip_liq_flux.data(), d.precip_ice_flux.data(), + d.cld_frac_r.data(), d.cld_frac_l.data(), d.cld_frac_i.data(), + d.liq_ice_exchange.data(), d.vap_liq_exchange.data(), + d.vap_ice_exchange.data(),d.qv_prev.data(),d.t_prev.data() ); +} + +int test_p3_init () { + using P3F = Functions; + + P3F::p3_init(); + return 0; +} + +int test_p3_ic () { + using P3F = Functions; + + const auto d = ic::Factory::create(ic::Factory::mixed); + + d->dt = 300.0; + P3F::p3_init(); + p3_main_wrap(*d); + return 0; +} + +} // namespace p3 +} // namespace scream diff --git a/components/eamxx/src/physics/p3/p3_main_wrap.hpp b/components/eamxx/src/physics/p3/tests/infra/p3_main_wrap.hpp similarity index 71% rename from components/eamxx/src/physics/p3/p3_main_wrap.hpp rename to components/eamxx/src/physics/p3/tests/infra/p3_main_wrap.hpp index 7c980fa8a5bb..c55007427cdb 100644 --- a/components/eamxx/src/physics/p3/p3_main_wrap.hpp +++ b/components/eamxx/src/physics/p3/tests/infra/p3_main_wrap.hpp @@ -8,15 +8,14 @@ namespace scream { namespace p3 { -struct FortranData; +struct P3Data; // Returns number of microseconds of p3_main execution -Int p3_main_wrap(const FortranData& d, bool use_fortran=false); +Int p3_main_wrap(const P3Data& d); int test_p3_init(); -int test_p3_ic(bool use_fortran); - +int test_p3_ic(); } // namespace p3 } // namespace scream diff --git a/components/eamxx/src/physics/p3/p3_functions_f90.cpp b/components/eamxx/src/physics/p3/tests/infra/p3_test_data.cpp similarity index 61% rename from components/eamxx/src/physics/p3/p3_functions_f90.cpp rename to components/eamxx/src/physics/p3/tests/infra/p3_test_data.cpp index 40d82d4f6529..2dff7183ec61 100644 --- a/components/eamxx/src/physics/p3/p3_functions_f90.cpp +++ b/components/eamxx/src/physics/p3/tests/infra/p3_test_data.cpp @@ -1,6 +1,6 @@ -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "ekat/kokkos/ekat_kokkos_types.hpp" -#include "p3_f90.hpp" +#include "p3_data.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "ekat/ekat_pack_kokkos.hpp" @@ -11,277 +11,10 @@ using scream::Real; using scream::Int; -// -// A C++ interface to micro_p3 fortran calls and vice versa -// - -extern "C" { - -void p3_init_a_c(Real* ice_table_vals, Real* collect_table_vals); - -void find_lookuptable_indices_1a_c(Int* dumi, Int* dumjj, Int* dumii, Int* dumzz, - Real* dum1, Real* dum4, Real* dum5, Real* dum6, - Real qi, Real ni, Real qm, Real rhop); - -void find_lookuptable_indices_1b_c(Int* dumj, Real* dum3, Real qr, Real nr); - -void access_lookup_table_c(Int dumjj, Int dumii, Int dumi, Int index, - Real dum1, Real dum4, Real dum5, Real* proc); - -void access_lookup_table_coll_c(Int dumjj, Int dumii, Int dumj, Int dumi, Int index, - Real dum1, Real dum3, Real dum4, Real dum5, Real* proc); - -void back_to_cell_average_c(Real cld_frac_l_, Real cld_frac_r_, Real cld_frac_i_, - Real* qc2qr_accret_tend_, Real* qr2qv_evap_tend_, Real* qc2qr_autoconv_tend_, - Real* nc_accret_tend_, Real* nc_selfcollect_tend_, Real* nc2nr_autoconv_tend_, - Real* nr_selfcollect_tend_, Real* nr_evap_tend_, Real* ncautr_, - Real* qi2qv_sublim_tend_, - Real* nr_ice_shed_tend_, Real* qc2qi_hetero_freeze_tend_, Real* qr2qi_collect_tend_, - Real* qc2qr_ice_shed_tend_, Real* qi2qr_melt_tend_, Real* qc2qi_collect_tend_, - Real* qr2qi_immers_freeze_tend_, Real* ni2nr_melt_tend_, Real* nc_collect_tend_, - Real* ncshdc_, Real* nc2ni_immers_freeze_tend_, Real* nr_collect_tend_, - Real* ni_selfcollect_tend_, Real* qv2qi_vapdep_tend_, Real* nr2ni_immers_freeze_tend_, - Real* ni_sublim_tend_, Real* qv2qi_nucleat_tend_, Real* ni_nucleat_tend_, - Real* qc2qi_berg_tend_); - -void cloud_water_conservation_c(Real qc, Real dt, Real* qc2qr_autoconv_tend, Real* qc2qr_accret_tend, Real* qc2qi_collect_tend, - Real* qc2qi_hetero_freeze_tend, Real* qc2qr_ice_shed_tend, Real* qc2qi_berg_tend, Real* qi2qv_sublim_tend, Real* qv2qi_vapdep_tend); - -void rain_water_conservation_c(Real qr, Real qc2qr_autoconv_tend, Real qc2qr_accret_tend, Real qi2qr_melt_tend, Real qc2qr_ice_shed_tend, - Real dt, Real* qr2qv_evap_tend, Real* qr2qi_collect_tend, Real* qr2qi_immers_freeze_tend); - -void ice_water_conservation_c(Real qi, Real qv2qi_vapdep_tend, Real qv2qi_nucleat_tend, Real qc2qi_berg_tend, Real qr2qi_collect_tend, Real qc2qi_collect_tend, - Real qr2qi_immers_freeze_tend, Real qc2qi_hetero_freeze_tend, Real dt, Real* qi2qv_sublim_tend, Real* qi2qr_melt_tend); - -void get_cloud_dsd2_c(Real qc, Real* nc, Real* mu_c, Real rho, Real* nu, Real* lamc, - Real* cdist, Real* cdist1); - -void get_rain_dsd2_c(Real qr, Real* nr, Real* mu_r, Real* lamr, Real* cdistr, Real* logn0r); - -void calc_rime_density_c(Real T_atm, Real rhofaci, Real table_val_qi_fallspd, Real acn, - Real lamc, Real mu_c, Real qc_incld, Real qc2qi_collect_tend, - Real* vtrmi1, Real* rho_qm_cloud); - -void cldliq_immersion_freezing_c(Real T_atm, Real lamc, Real mu_c, Real cdist1, - Real qc_incld, Real inv_qc_relvar, Real* qc2qi_hetero_freeze_tend, Real* nc2ni_immers_freeze_tend); - -void rain_immersion_freezing_c(Real T_atm, Real lamr, Real mu_r, Real cdistr, - Real qr_incld, Real* qr2qi_immers_freeze_tend, Real* nr2ni_immers_freeze_tend); - -void droplet_self_collection_c(Real rho, Real inv_rho, Real qc_incld, Real mu_c, - Real nu, Real nc2nr_autoconv_tend, Real* nc_accret_tend); - -void cloud_rain_accretion_c(Real rho, Real inv_rho, Real qc_incld, Real nc_incld, - Real qr_incld, Real inv_qc_relvar, Real* qc2qr_accret_tend, Real* nc_accret_tend); - -void cloud_water_autoconversion_c(Real rho, Real qc_incld, Real nc_incld, Real inv_qc_relvar, Real* qc2qr_autoconv_tend, Real* nc2nr_autoconv_tend, Real* ncautr); - -void rain_self_collection_c(Real rho, Real qr_incld, Real nr_incld, Real* nr_selfcollect_tend); - -void impose_max_total_ni_c(Real* ni_local, Real max_total_ni, Real inv_rho_local); - -void ice_melting_c(Real rho,Real T_atm,Real pres,Real rhofaci,Real table_val_qi2qr_melting,Real table_val_qi2qr_vent_melt, - Real latent_heat_vapor,Real latent_heat_fusion,Real dv,Real sc,Real mu,Real kap,Real qv,Real qi_incld, - Real ni_incld,Real* qi2qr_melt_tend,Real* ni2nr_melt_tend); - -void calc_first_order_upwind_step_c(Int kts, Int kte, Int kdir, Int kbot, Int k_qxtop, Real dt_sub, Real* rho, - Real* inv_rho, Real* inv_dz, Int num_arrays, Real** fluxes, Real** vs, Real** qnx); - -void generalized_sedimentation_c(Int kts, Int kte, Int kdir, Int k_qxtop, Int* k_qxbot, Int kbot, Real Co_max, - Real* dt_left, Real* prt_accum, Real* inv_dz, Real* inv_rho, Real* rho, - Int num_arrays, Real** vs, Real** fluxes, Real** qnx); -void cloud_sedimentation_c( - Int kts, Int kte, Int ktop, Int kbot, Int kdir, - Real* qc_incld, Real* rho, Real* inv_rho, Real* cld_frac_l, Real* acn, Real* inv_dz, - Real dt, Real inv_dt, bool do_predict_nc, - Real* qc, Real* nc, Real* nc_incld, Real* mu_c, Real* lamc, Real* precip_liq_surf, Real* qc_tend, Real* nc_tend); - -void ice_sedimentation_c( - Int kts, Int kte, Int ktop, Int kbot, Int kdir, - Real* rho, Real* inv_rho, Real* rhofaci, Real* cld_frac_i, Real* inv_dz, - Real dt, Real inv_dt, - Real* qi, Real* qi_incld, Real* ni, Real* qm, Real* qm_incld, Real* bm, Real* bm_incld, - Real* ni_incld, Real* precip_ice_surf, Real* qi_tend, Real* ni_tend); - -void rain_sedimentation_c( - Int kts, Int kte, Int ktop, Int kbot, Int kdir, - Real* qr_incld, Real* rho, Real* inv_rho, Real* rhofacr, Real* cld_frac_r, Real* inv_dz, - Real dt, Real inv_dt, - Real* qr, Real* nr, Real* nr_incld, Real* mu_r, Real* lamr, Real* precip_liq_surf, Real* precip_liq_flux, Real* qr_tend, Real* nr_tend); - -void calc_bulk_rho_rime_c(Real qi_tot, Real* qi_rim, Real* bi_rim, Real* rho_rime); - -void homogeneous_freezing_c( - Int kts, Int kte, Int ktop, Int kbot, Int kdir, - Real* T_atm, Real* inv_exner, Real* latent_heat_fusion, - Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* th_atm); - -void get_time_space_phys_variables_c(Real T_atm, Real pres, Real rho, Real latent_heat_vapor, Real latent_heat_sublim, Real qv_sat_l, Real qv_sat_i, - Real* mu, Real* dv, Real* sc, Real* dqsdt, Real* dqsidt, Real* ab, Real* abi, Real* kap, Real* eii); - -void update_prognostic_ice_c( - Real qc2qi_hetero_freeze_tend, Real qc2qi_collect_tend, Real qc2qr_ice_shed_tend, Real nc_collect_tend, Real nc2ni_immers_freeze_tend, Real ncshdc, - Real qr2qi_collect_tend, Real nr_collect_tend, Real qr2qi_immers_freeze_tend, Real nr2ni_immers_freeze_tend, Real nr_ice_shed_tend, - Real qi2qr_melt_tend, Real ni2nr_melt_tend, Real qi2qv_sublim_tend, Real qv2qi_vapdep_tend, Real qv2qi_nucleat_tend, Real ni_nucleat_tend, - Real ni_selfcollect_tend, Real ni_sublim_tend, Real qc2qi_berg_tend, Real inv_exner, Real latent_heat_sublim, Real latent_heat_fusion, - bool do_predict_nc, bool log_wetgrowth, Real dt, Real nmltratio, - Real rho_qm_cloud, Real* th_atm, Real* qv, Real* qi, Real* ni, Real* qm, - Real* bm, Real* qc, Real* nc, Real* qr, Real* nr); - -void evaporate_rain_c( Real qr_incld, Real qc_incld, Real nr_incld, Real qi_incld, - Real cld_frac_l, Real cld_frac_r, Real qv, Real qv_prev, - Real qv_sat_l, Real qv_sat_i, Real ab, Real abi, - Real epsr, Real epsi_tot, Real t, Real t_prev, - Real latent_heat_sublim, Real dqsdt, Real dt, - Real* qr2qv_evap_tend, Real* nr_evap_tend); - -void update_prognostic_liquid_c( - Real qc2qr_accret_tend, Real nc_accret_tend, Real qc2qr_autoconv_tend, Real nc2nr_autoconv_tend, Real ncautr, - Real nc_selfcollect_tend, Real qr2qv_evap_tend, Real nr_evap_tend, Real nr_selfcollect_tend , bool do_predict_nc, bool do_prescribed_CCN, - Real inv_rho, Real inv_exner, Real latent_heat_vapor, Real dt, Real* th_atm, Real* qv, - Real* qc, Real* nc, Real* qr, Real* nr); - -void ice_deposition_sublimation_c(Real qi_incld, Real ni_incld, Real t_atm, Real qv_sat_l, Real qv_sat_i, Real epsi, Real abi, Real qv, Real inv_dt, Real* qidep, Real* qi2qv_sublim_tend, Real* ni_sublim_tend, Real* qiberg); - -void compute_rain_fall_velocity_c(Real qr_incld, Real rhofacr, - Real* nr_incld, Real* mu_r, Real* lamr, Real* V_qr, Real* V_nr); - -void ice_cldliq_collection_c(Real rho, Real temp, Real rhofaci, Real table_val_qc2qi_collect, - Real qi_incld,Real qc_incld, Real ni_incld, Real nc_incld, - Real* qc2qi_collect_tend, Real* nc_collect_tend, Real* qc2qr_ice_shed_tend, Real* ncshdc); - -void ice_rain_collection_c(Real rho, Real temp, Real rhofaci, Real logn0r, Real table_val_nr_collect, Real table_val_qr2qi_collect, - Real qi_incld, Real ni_incld, Real qr_incld, Real* qr2qi_collect_tend, Real* nr_collect_tend); - - -void ice_self_collection_c(Real rho, Real rhofaci, Real table_val_ni_self_collect, Real eii, - Real qm_incld, Real qi_incld, Real ni_incld, Real* ni_selfcollect_tend); - -void ice_relaxation_timescale_c(Real rho, Real temp, Real rhofaci, Real table_val_qi2qr_melting, Real table_val_qi2qr_vent_melt, - Real dv, Real mu, Real sc, Real qi_incld, Real ni_incld, - Real* epsi, Real* epsi_tot); - -void calc_liq_relaxation_timescale_c(Real rho, Real f1r, Real f2r, Real dv, - Real mu, Real sc, Real mu_r, Real lamr, - Real cdistr, Real cdist, Real qr_incld, - Real qc_incld, Real* epsr, Real* epsc); - -void ice_nucleation_c(Real temp, Real inv_rho, Real ni, Real ni_activated, - Real qv_supersat_i, Real inv_dt, bool do_predict_nc, bool do_prescribed_CCN, - Real* qv2qi_nucleat_tend, Real* ni_nucleat_tend); - -void ice_cldliq_wet_growth_c(Real rho, Real temp, Real pres, Real rhofaci, Real table_val_qi2qr_melting, - Real table_val_qi2qr_vent_melt, Real latent_heat_vapor, Real latent_heat_fusion, Real dv, - Real kap, Real mu, Real sc, Real qv, Real qc_incld, - Real qi_incld, Real ni_incld, Real qr_incld, bool* log_wetgrowth, - Real* qr2qi_collect_tend, Real* qc2qi_collect_tend, Real* qc_growth_rate, Real* nr_ice_shed_tend, Real* qc2qr_ice_shed_tend); - -void get_latent_heat_c(Int its, Int ite, Int kts, Int kte, Real* s, Real* v, Real* f); - -Real subgrid_variance_scaling_c(Real relvar, Real expon); - -void check_values_c(Real* qv, Real* temp, Int kts, Int kte, Int timestepcount, - Int force_abort, Int source_ind, Real* col_loc); - -void calculate_incloud_mixingratios_c(Real qc, Real qr, Real qi, Real qm, Real nc, Real nr, Real ni, Real bm, - Real inv_cld_frac_l, Real inv_cld_frac_i, Real inv_cld_frac_r, - Real* qc_incld, Real* qr_incld, Real* qi_incld, Real* qm_incld, - Real* nc_incld, Real* nr_incld, Real* ni_incld, Real* bm_incld); - -void p3_main_part1_c( - Int kts, Int kte, Int kbot, Int ktop, Int kdir, - bool do_predict_nc, bool do_prescribed_CCN, - Real dt, - Real* pres, Real* dpres, Real* dz, Real* nc_nuceat_tend, Real* nccn_prescribed, Real* inv_exner, Real* exner, Real* inv_cld_frac_l, Real* inv_cld_frac_i, - Real* inv_cld_frac_r, Real* latent_heat_vapor, Real* latent_heat_sublim, Real* latent_heat_fusion, - Real* T_atm, Real* rho, Real* inv_rho, Real* qv_sat_l, Real* qv_sat_i, Real* qv_supersat_i, Real* rhofacr, Real* rhofaci, - Real* acn, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* qc_incld, Real* qr_incld, Real* qi_incld, - Real* qm_incld, Real* nc_incld, Real* nr_incld, Real* ni_incld, Real* bm_incld, - bool* is_nucleat_possible, bool* is_hydromet_present); - -void p3_main_part2_c( - Int kts, Int kte, Int kbot, Int ktop, Int kdir, bool do_predict_nc, bool do_prescribed_CCN, Real dt, Real inv_dt, - Real* pres, Real* inv_exner, Real* inv_cld_frac_l, Real* inv_cld_frac_i, - Real* inv_cld_frac_r, Real* ni_activated, Real* inv_qc_relvar, Real* cld_frac_i, Real* cld_frac_l, Real* cld_frac_r, Real* qv_prev, Real* t_prev, - Real* T_atm, Real* rho, Real* inv_rho, Real* qv_sat_l, Real* qv_sat_i, Real* qv_supersat_i, Real* rhofaci, Real* acn, - Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, - Real* qm, Real* bm, Real* latent_heat_vapor, Real* latent_heat_sublim, Real* latent_heat_fusion, Real* qc_incld, - Real* qr_incld, Real* qi_incld, Real* qm_incld, Real* nc_incld, Real* nr_incld, - Real* ni_incld, Real* bm_incld, Real* mu_c, Real* nu, Real* lamc, Real* cdist, Real* cdist1, - Real* cdistr, Real* mu_r, Real* lamr, Real* logn0r, Real* qv2qi_depos_tend, Real* precip_total_tend, - Real* nevapr, Real* qr_evap_tend, Real* vap_liq_exchange, Real* vap_ice_exchange, Real* liq_ice_exchange, Real* pratot, - Real* prctot, bool* is_hydromet_present); - -void p3_main_part3_c( - Int kts, Int kte, Int kbot, Int ktop, Int kdir, - Real* inv_exner, Real* cld_frac_l, Real* cld_frac_r, Real* cld_frac_i, - Real* rho, Real* inv_rho, Real* rhofaci, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, - Real* qi, Real* ni, Real* qm, Real* bm, Real* latent_heat_vapor, Real* latent_heat_sublim, - Real* mu_c, Real* nu, Real* lamc, Real* mu_r, Real* lamr, Real* vap_liq_exchange, - Real* ze_rain, Real* ze_ice, Real* diag_vm_qi, Real* diag_eff_radius_qi, Real* diag_diam_qi, Real* rho_qi, Real* diag_equiv_reflectivity, Real* diag_eff_radius_qc, Real* diag_eff_radius_qr); - -void p3_main_c( - Real* qc, Real* nc, Real* qr, Real* nr, Real* th_atm, Real* qv, Real dt, - Real* qi, Real* qm, Real* ni, Real* bm, Real* pres, Real* dz, - Real* nc_nuceat_tend, Real* nccn_prescribed, Real* ni_activated, Real* inv_qc_relvar, Int it, Real* precip_liq_surf, - Real* precip_ice_surf, Int its, Int ite, Int kts, Int kte, Real* diag_eff_radius_qc, - Real* diag_eff_radius_qi, Real* diag_eff_radius_qr, Real* rho_qi, bool do_predict_nc, bool do_prescribed, Real* dpres, Real* inv_exner, - Real* qv2qi_depos_tend, Real* precip_liq_flux, Real* precip_ice_flux, Real* cld_frac_r, Real* cld_frac_l, Real* cld_frac_i, - Real* liq_ice_exchange, Real* vap_liq_exchange, Real* vap_ice_exchange, Real* qv_prev, Real* t_prev, Real* elapsed_s); - -void ice_supersat_conservation_c(Real* qidep, Real* qinuc, Real cld_frac_i, Real qv, Real qv_sat_i, Real latent_heat_sublim, Real t_atm, Real dt, Real qi2qv_sublim_tend, Real qr2qv_evap_tend); -void nc_conservation_c(Real nc, Real nc_selfcollect_tend, Real dt, Real* nc_collect_tend, Real* nc2ni_immers_freeze_tend, Real* nc_accret_tend, Real* nc2nr_autoconv_tend); -void nr_conservation_c(Real nr, Real ni2nr_melt_tend, Real nr_ice_shed_tend, Real ncshdc, Real nc2nr_autoconv_tend, Real dt, Real nmltratio, Real* nr_collect_tend, Real* nr2ni_immers_freeze_tend, Real* nr_selfcollect_tend, Real* nr_evap_tend); -void ni_conservation_c(Real ni, Real ni_nucleat_tend, Real nr2ni_immers_freeze_tend, Real nc2ni_immers_freeze_tend, Real dt, Real* ni2nr_melt_tend, Real* ni_sublim_tend, Real* ni_selfcollect_tend); -void prevent_liq_supersaturation_c(Real pres, Real t_atm, Real qv, Real latent_heat_vapor, Real latent_heat_sublim, Real dt, Real qidep, Real qinuc, Real* qi2qv_sublim_tend, Real* qr2qv_evap_tend); -} // extern "C" : end _c decls namespace scream { namespace p3 { -// -// In all C++ -> Fortran bridge functions you should see p3_init(). P3 needs -// to be initialized since most of its function depend on global tables to be -// populated. The 'true' argument is to set p3 to use its fortran implementations -// instead of calling back to C++. We want this behavior since it doesn't make much -// sense for C++ to bridge over to fortran only to have fortran bridge back to C++. -// If the client wanted the C++ implementation, they should just call it directly. -// - -void p3_init_a(P3InitAFortranData& d) -{ - p3_init(); // need to initialize p3 first so that tables are loaded - p3_init_a_c(d.ice_table_vals.data(), d.collect_table_vals.data()); -} - -void find_lookuptable_indices_1a(LookupIceData& d) -{ - p3_init(); // need to initialize p3 first so that tables are loaded - find_lookuptable_indices_1a_c(&d.dumi, &d.dumjj, &d.dumii, &d.dumzz, - &d.dum1, &d.dum4, &d.dum5, &d.dum6, - d.qi, d.ni, d.qm, d.rhop); -} - -void find_lookuptable_indices_1b(LookupIceDataB& d) -{ - p3_init(); - find_lookuptable_indices_1b_c(&d.dumj, &d.dum3, d.qr, d.nr); -} - -void access_lookup_table(AccessLookupTableData& d) -{ - p3_init(); // need to initialize p3 first so that tables are loaded - access_lookup_table_c(d.lid.dumjj, d.lid.dumii, d.lid.dumi, d.index, - d.lid.dum1, d.lid.dum4, d.lid.dum5, &d.proc); -} - -void access_lookup_table_coll(AccessLookupTableCollData& d) -{ - p3_init(); // need to initialize p3 first so that tables are loaded - access_lookup_table_coll_c(d.lid.dumjj, d.lid.dumii, d.lidb.dumj, d.lid.dumi, d.index, - d.lid.dum1, d.lidb.dum3, d.lid.dum4, d.lid.dum5, &d.proc); -} - void BackToCellAverageData::randomize(std::mt19937_64& engine) { // Populate the struct with numbers between 0 and 1. @@ -322,155 +55,6 @@ void BackToCellAverageData::randomize(std::mt19937_64& engine) qc2qi_berg_tend = data_dist(engine); } -void back_to_cell_average(BackToCellAverageData& d) -{ - p3_init(); - back_to_cell_average_c(d.cld_frac_l, d.cld_frac_r, d.cld_frac_i, &d.qc2qr_accret_tend, &d.qr2qv_evap_tend, - &d.qc2qr_autoconv_tend, &d.nc_accret_tend, &d.nc_selfcollect_tend, &d.nc2nr_autoconv_tend, &d.nr_selfcollect_tend, &d.nr_evap_tend, &d.ncautr, - &d.qi2qv_sublim_tend, &d.nr_ice_shed_tend, &d.qc2qi_hetero_freeze_tend, &d.qr2qi_collect_tend, &d.qc2qr_ice_shed_tend, - &d.qi2qr_melt_tend, &d.qc2qi_collect_tend, &d.qr2qi_immers_freeze_tend, &d.ni2nr_melt_tend, &d.nc_collect_tend, &d.ncshdc, &d.nc2ni_immers_freeze_tend, - &d.nr_collect_tend, &d.ni_selfcollect_tend, &d.qv2qi_vapdep_tend, &d.nr2ni_immers_freeze_tend, &d.ni_sublim_tend, &d.qv2qi_nucleat_tend, &d.ni_nucleat_tend, - &d.qc2qi_berg_tend); -} - -void calc_rime_density(CalcRimeDensityData& d) -{ - p3_init(); - calc_rime_density_c(d.T_atm, d.rhofaci, d.table_val_qi_fallspd, d.acn, d.lamc, d.mu_c, - d.qc_incld, d.qc2qi_collect_tend, &d.vtrmi1, &d.rho_qm_cloud); -} - -void cldliq_immersion_freezing(CldliqImmersionFreezingData& d) -{ - p3_init(); - cldliq_immersion_freezing_c(d.T_atm, d.lamc, d.mu_c, d.cdist1, d.qc_incld, d.inv_qc_relvar, - &d.qc2qi_hetero_freeze_tend, &d.nc2ni_immers_freeze_tend); -} - -LatentHeatData::LatentHeatData(Int kts_, Int kte_, Int its_, Int ite_) : - PhysicsTestData( { {(ite_ - its_) + 1, (kte_ - kts_) + 1} }, - { {&v, &s, &f} }), - its(its_), ite(ite_), kts(kts_), kte(kte_) -{} - -void get_latent_heat(LatentHeatData& d) -{ - p3_init(); - d.transpose(); - get_latent_heat_c(d.its, d.ite, d.kts, d.kte, d.v, d.s, d.f); - d.transpose(); -} - -void droplet_self_collection(DropletSelfCollectionData& d) -{ - p3_init(); - droplet_self_collection_c(d.rho, d.inv_rho, d.qc_incld, d.mu_c, d.nu, d.nc2nr_autoconv_tend, - &d.nc_selfcollect_tend); -} - -void rain_immersion_freezing(RainImmersionFreezingData& d) -{ - p3_init(); - rain_immersion_freezing_c(d.T_atm, d.lamr, d.mu_r, d.cdistr, d.qr_incld, - &d.qr2qi_immers_freeze_tend, &d.nr2ni_immers_freeze_tend); -} - -void cloud_rain_accretion(CloudRainAccretionData& d) -{ - p3_init(); - cloud_rain_accretion_c(d.rho, d.inv_rho, d.qc_incld, d.nc_incld, d.qr_incld, d.inv_qc_relvar, - &d.qc2qr_accret_tend, &d.nc_accret_tend); -} - -void cloud_water_conservation(CloudWaterConservationData& d){ - p3_init(); - cloud_water_conservation_c(d.qc, d.dt, &d.qc2qr_autoconv_tend, &d.qc2qr_accret_tend, &d.qc2qi_collect_tend, &d.qc2qi_hetero_freeze_tend, - &d.qc2qr_ice_shed_tend, &d.qc2qi_berg_tend, &d.qi2qv_sublim_tend, &d.qv2qi_vapdep_tend); -} - -void rain_water_conservation(RainWaterConservationData& d){ - p3_init(); - rain_water_conservation_c(d.qr, d.qc2qr_autoconv_tend, d.qc2qr_accret_tend, d.qi2qr_melt_tend, d.qc2qr_ice_shed_tend, - d.dt, &d.qr2qv_evap_tend, &d.qr2qi_collect_tend, &d.qr2qi_immers_freeze_tend); -} - -void ice_water_conservation(IceWaterConservationData& d){ - p3_init(); - ice_water_conservation_c(d.qi, d.qv2qi_vapdep_tend, d.qv2qi_nucleat_tend, d.qc2qi_berg_tend, d.qr2qi_collect_tend, d.qc2qi_collect_tend, d.qr2qi_immers_freeze_tend, - d.qc2qi_hetero_freeze_tend, d.dt, &d.qi2qv_sublim_tend, &d.qi2qr_melt_tend); -} - -void cloud_water_autoconversion(CloudWaterAutoconversionData& d){ - p3_init(); - cloud_water_autoconversion_c(d.rho, d.qc_incld, d.nc_incld, d.inv_qc_relvar, - &d.qc2qr_autoconv_tend, &d.nc2nr_autoconv_tend, &d.ncautr); -} - -void rain_self_collection(RainSelfCollectionData& d){ - p3_init(); - rain_self_collection_c(d.rho, d.qr_incld, d.nr_incld, &d.nr_selfcollect_tend); -} - -void impose_max_total_ni(ImposeMaxTotalNiData& d){ - p3_init(); - impose_max_total_ni_c(&d.ni_local, d.max_total_ni, d.inv_rho_local); -} - -void get_cloud_dsd2(GetCloudDsd2Data& d) -{ - p3_init(); - Real nc_in = d.nc_in; - get_cloud_dsd2_c(d.qc, &nc_in, &d.mu_c, d.rho, &d.nu, &d.lamc, &d.cdist, &d.cdist1); - d.nc_out = nc_in; -} - -void get_rain_dsd2(GetRainDsd2Data& d) -{ - p3_init(); - Real nr_in = d.nr_in; - get_rain_dsd2_c(d.qr, &nr_in, &d.mu_r, &d.lamr, &d.cdistr, &d.logn0r); - d.nr_out = nr_in; -} - -void ice_cldliq_collection(IceCldliqCollectionData& d) -{ - p3_init(); - ice_cldliq_collection_c(d.rho, d.temp, d.rhofaci, d.table_val_qc2qi_collect, - d.qi_incld, d.qc_incld, d.ni_incld, d.nc_incld, - &d.qc2qi_collect_tend, &d.nc_collect_tend, &d.qc2qr_ice_shed_tend, &d.ncshdc); -} - -void ice_rain_collection(IceRainCollectionData& d) -{ - p3_init(); - ice_rain_collection_c(d.rho, d.temp, d.rhofaci, d.logn0r, d.table_val_nr_collect, d.table_val_qr2qi_collect, - d.qi_incld, d.ni_incld, d.qr_incld, - &d.qr2qi_collect_tend, &d.nr_collect_tend); -} - -void ice_self_collection(IceSelfCollectionData& d) -{ - p3_init(); - ice_self_collection_c(d.rho, d.rhofaci, d.table_val_ni_self_collect, d.eii, d.qm_incld, - d.qi_incld, d.ni_incld, - &d.ni_selfcollect_tend); -} - -void get_time_space_phys_variables(GetTimeSpacePhysVarsData& d) -{ - p3_init(); - get_time_space_phys_variables_c(d.T_atm, d.pres, d.rho, d.latent_heat_vapor, d.latent_heat_sublim, d.qv_sat_l, d.qv_sat_i, &d.mu, &d.dv, - &d.sc, &d.dqsdt, &d.dqsidt, &d.ab, &d.abi, &d.kap, &d.eii); -} - -void ice_relaxation_timescale(IceRelaxationData& d) -{ - p3_init(); - ice_relaxation_timescale_c(d.rho, d.temp, d.rhofaci, d.table_val_qi2qr_melting, d.table_val_qi2qr_vent_melt, - d.dv, d.mu, d.sc, d.qi_incld, d.ni_incld, - &d.epsi, &d.epsi_tot); -} - void CalcLiqRelaxationData::randomize(std::mt19937_64& engine) { // Populate the struct's input fields with numbers between 0 and 1. @@ -489,31 +73,6 @@ void CalcLiqRelaxationData::randomize(std::mt19937_64& engine) qc_incld = data_dist(engine); } -void calc_liq_relaxation_timescale(CalcLiqRelaxationData& d) -{ - p3_init(); - calc_liq_relaxation_timescale_c(d.rho, d.f1r, d.f2r, d.dv, d.mu, d.sc, d.mu_r, - d.lamr, d.cdistr, d.cdist, d.qr_incld, d.qc_incld, &d.epsr, &d.epsc); -} - -void ice_nucleation(IceNucleationData& d) -{ - p3_init(); - ice_nucleation_c(d.temp, d.inv_rho, d.ni, d.ni_activated, - d.qv_supersat_i, d.inv_dt, d.do_predict_nc, d.do_prescribed_CCN, &d.qv2qi_nucleat_tend, &d.ni_nucleat_tend); -} - -void ice_cldliq_wet_growth(IceWetGrowthData& d) -{ - p3_init(); - - ice_cldliq_wet_growth_c(d.rho, d.temp, d.pres, d.rhofaci, d.table_val_qi2qr_melting, - d.table_val_qi2qr_vent_melt, d.latent_heat_vapor, d.latent_heat_fusion, d.dv, - d.kap, d.mu, d.sc, d.qv, d.qc_incld, - d.qi_incld, d.ni_incld, d.qr_incld, &d.log_wetgrowth, - &d.qr2qi_collect_tend, &d.qc2qi_collect_tend, &d.qc_growth_rate, &d.nr_ice_shed_tend, &d.qc2qr_ice_shed_tend); -} - CheckValuesData::CheckValuesData( Int kts_, Int kte_, Int timestepcount_, Int source_ind_, bool force_abort_) : PhysicsTestData( { {(kte_-kts_)+1} }, @@ -523,57 +82,6 @@ CheckValuesData::CheckValuesData( EKAT_REQUIRE_MSG(nk() >= 3 || (kte == 1 && kts == 1), "nk too small to use for col_loc"); } -void check_values(CheckValuesData& d) -{ - p3_init(); - check_values_c(d.qv, d.temp, d.kts, d.kte, d.timestepcount, - d.force_abort, d.source_ind, d.col_loc); -} - -void calculate_incloud_mixingratios(IncloudMixingData& d) -{ - p3_init(); - - calculate_incloud_mixingratios_c(d.qc, d.qr, d.qi, d.qm, d.nc, d.nr, d.ni, d.bm, d.inv_cld_frac_l, d.inv_cld_frac_i, d.inv_cld_frac_r, - &d.qc_incld, &d.qr_incld, &d.qi_incld, &d.qm_incld, - &d.nc_incld, &d.nr_incld, &d.ni_incld, &d.bm_incld); - -} - -void update_prognostic_ice(P3UpdatePrognosticIceData& d){ - p3_init(); - update_prognostic_ice_c(d.qc2qi_hetero_freeze_tend, d.qc2qi_collect_tend, d.qc2qr_ice_shed_tend, d.nc_collect_tend, d.nc2ni_immers_freeze_tend, d.ncshdc, - d.qr2qi_collect_tend, d.nr_collect_tend, d.qr2qi_immers_freeze_tend, d.nr2ni_immers_freeze_tend, d.nr_ice_shed_tend, - d.qi2qr_melt_tend, d.ni2nr_melt_tend, d.qi2qv_sublim_tend, d.qv2qi_vapdep_tend, d.qv2qi_nucleat_tend, d.ni_nucleat_tend, - d.ni_selfcollect_tend, d.ni_sublim_tend, d.qc2qi_berg_tend, d.inv_exner, d.latent_heat_sublim, d.latent_heat_fusion, - d.do_predict_nc, d.log_wetgrowth, d.dt, d.nmltratio, - d.rho_qm_cloud, &d.th_atm, &d.qv, &d.qi, &d.ni, &d.qm, - &d.bm, &d.qc, &d.nc, &d.qr, &d.nr); -} - -void evaporate_rain(EvapRainData& d) -{ - p3_init(); - evaporate_rain_c(d.qr_incld,d.qc_incld,d.nr_incld,d.qi_incld, - d.cld_frac_l,d.cld_frac_r,d.qv,d.qv_prev,d.qv_sat_l,d.qv_sat_i, - d.ab,d.abi,d.epsr,d.epsi_tot,d.t,d.t_prev,d.latent_heat_sublim,d.dqsdt,d.dt, - &d.qr2qv_evap_tend,&d.nr_evap_tend); -} - -void update_prognostic_liquid(P3UpdatePrognosticLiqData& d){ - p3_init(); - update_prognostic_liquid_c(d.qc2qr_accret_tend, d.nc_accret_tend, d.qc2qr_autoconv_tend, d.nc2nr_autoconv_tend, d.ncautr, - d.nc_selfcollect_tend, d. qr2qv_evap_tend, d.nr_evap_tend, d.nr_selfcollect_tend , d.do_predict_nc, d.do_prescribed_CCN, - d.inv_rho, d.inv_exner, d.latent_heat_vapor, d.dt, &d.th_atm, &d.qv, - &d.qc, &d.nc, &d.qr, &d.nr); -} - -void ice_deposition_sublimation(IceDepositionSublimationData& d) -{ - p3_init(); - ice_deposition_sublimation_c(d.qi_incld, d.ni_incld, d.T_atm, d.qv_sat_l, d.qv_sat_i, d.epsi, d.abi, d.qv, d.inv_dt, &d.qv2qi_vapdep_tend, &d.qi2qv_sublim_tend, &d.ni_sublim_tend, &d.qc2qi_berg_tend); -} - CalcUpwindData::CalcUpwindData( Int kts_, Int kte_, Int kdir_, Int kbot_, Int k_qxtop_, Int num_arrays_, Real dt_sub_) : PhysicsTestData({ {(kte_ - kts_)+1, num_arrays_}, {(kte_ - kts_)+1} }, @@ -594,15 +102,6 @@ void CalcUpwindData::convert_to_ptr_arr(std::vector& mem_space, Real**& f qnx_ = mem_space.data() + num_arrays*2; } -void calc_first_order_upwind_step(CalcUpwindData& d) -{ - p3_init(); - std::vector tmp; - Real** fluxes, **vs, **qnx; - d.convert_to_ptr_arr(tmp, fluxes, vs, qnx); - calc_first_order_upwind_step_c(d.kts, d.kte, d.kdir, d.kbot, d.k_qxtop, d.dt_sub, d.rho, d.inv_rho, d.inv_dz, d.num_arrays, fluxes, vs, qnx); -} - GenSedData::GenSedData( Int kts_, Int kte_, Int kdir_, Int k_qxtop_, Int k_qxbot_, Int kbot_, Real Co_max_, Real dt_left_, Real prt_accum_, Int num_arrays_) : @@ -610,17 +109,6 @@ GenSedData::GenSedData( Co_max(Co_max_), k_qxbot(k_qxbot_), dt_left(dt_left_), prt_accum(prt_accum_) { } -void generalized_sedimentation(GenSedData& d) -{ - p3_init(); - std::vector tmp; - Real** fluxes, **vs, **qnx; - d.convert_to_ptr_arr(tmp, fluxes, vs, qnx); - generalized_sedimentation_c(d.kts, d.kte, d.kdir, d.k_qxtop, &d.k_qxbot, d.kbot, d.Co_max, - &d.dt_left, &d.prt_accum, d.inv_dz, d.inv_rho, d.rho, - d.num_arrays, fluxes, vs, qnx); -} - CloudSedData::CloudSedData( Int kts_, Int kte_, Int ktop_, Int kbot_, Int kdir_, Real dt_, Real inv_dt_, bool do_predict_nc_, Real precip_liq_surf_) : @@ -630,15 +118,6 @@ CloudSedData::CloudSedData( dt(dt_), inv_dt(inv_dt_), do_predict_nc(do_predict_nc_), precip_liq_surf(precip_liq_surf_) {} -void cloud_sedimentation(CloudSedData& d) -{ - p3_init(); - cloud_sedimentation_c(d.kts, d.kte, d.ktop, d.kbot, d.kdir, - d.qc_incld, d.rho, d.inv_rho, d.cld_frac_l, d.acn, d.inv_dz, - d.dt, d.inv_dt, d.do_predict_nc, - d.qc, d.nc, d.nc_incld, d.mu_c, d.lamc, &d.precip_liq_surf, d.qc_tend, d.nc_tend); -} - IceSedData::IceSedData( Int kts_, Int kte_, Int ktop_, Int kbot_, Int kdir_, Real dt_, Real inv_dt_, Real precip_ice_surf_) : @@ -648,15 +127,6 @@ IceSedData::IceSedData( dt(dt_), inv_dt(inv_dt_), precip_ice_surf(precip_ice_surf_) {} -void ice_sedimentation(IceSedData& d) -{ - p3_init(); - ice_sedimentation_c(d.kts, d.kte, d.ktop, d.kbot, d.kdir, - d.rho, d.inv_rho, d.rhofaci, d.cld_frac_i, d.inv_dz, d.dt, d.inv_dt, - d.qi, d.qi_incld, d.ni, d.qm, d.qm_incld, d.bm, d.bm_incld, d.ni_incld, - &d.precip_ice_surf, d.qi_tend, d.ni_tend); -} - RainSedData::RainSedData( Int kts_, Int kte_, Int ktop_, Int kbot_, Int kdir_, Real dt_, Real inv_dt_, Real precip_liq_surf_) : @@ -666,21 +136,6 @@ RainSedData::RainSedData( dt(dt_), inv_dt(inv_dt_), precip_liq_surf(precip_liq_surf_) {} -void rain_sedimentation(RainSedData& d) -{ - p3_init(); - rain_sedimentation_c(d.kts, d.kte, d.ktop, d.kbot, d.kdir, - d.qr_incld, d.rho, d.inv_rho, d.rhofacr, d.cld_frac_r, d.inv_dz, - d.dt, d.inv_dt, - d.qr, d.nr, d.nr_incld, d.mu_r, d.lamr, &d.precip_liq_surf, d.precip_liq_flux, d.qr_tend, d.nr_tend); -} - -void calc_bulk_rho_rime(CalcBulkRhoRimeData& d) -{ - p3_init(); - calc_bulk_rho_rime_c(d.qi_tot, &d.qi_rim, &d.bi_rim, &d.rho_rime); -} - HomogeneousFreezingData::HomogeneousFreezingData( Int kts_, Int kte_, Int ktop_, Int kbot_, Int kdir_) : PhysicsTestData( { {(kte_ - kts_) + 1} }, @@ -688,36 +143,9 @@ HomogeneousFreezingData::HomogeneousFreezingData( kts(kts_), kte(kte_), ktop(ktop_), kbot(kbot_), kdir(kdir_) {} -void homogeneous_freezing(HomogeneousFreezingData& d) -{ - p3_init(); - homogeneous_freezing_c(d.kts, d.kte, d.ktop, d.kbot, d.kdir, - d.T_atm, d.inv_exner, d.latent_heat_fusion, - d.qc, d.nc, d.qr, d.nr, d.qi, d.ni, d.qm, d.bm, d.th_atm); -} - -void ice_melting(IceMeltingData& d){ - p3_init(); - ice_melting_c(d.rho,d.T_atm,d.pres,d.rhofaci,d.table_val_qi2qr_melting,d.table_val_qi2qr_vent_melt, - d.latent_heat_vapor,d.latent_heat_fusion,d.dv,d.sc,d.mu,d.kap, - d.qv,d.qi_incld,d.ni_incld,&d.qi2qr_melt_tend,&d.ni2nr_melt_tend); -} - -Real subgrid_variance_scaling(SubgridVarianceScalingData& d){ - p3_init(); - return subgrid_variance_scaling_c(d.relvar,d.expon); -} - -void compute_rain_fall_velocity(ComputeRainFallVelocityData& d) -{ - p3_init(); - compute_rain_fall_velocity_c(d.qr_incld, d.rhofacr, - &d.nr_incld, &d.mu_r, &d.lamr, &d.V_qr, &d.V_nr); -} - P3MainPart1Data::P3MainPart1Data( Int kts_, Int kte_, Int kbot_, Int ktop_, Int kdir_, - bool do_predict_nc_, bool do_prescribed_CCN_, Real dt_) : + bool do_predict_nc_, bool do_prescribed_CCN_, Real dt_, bool, bool) : PhysicsTestData( { {(kte_ - kts_) + 1} }, { { &pres, &dpres, &dz, &nc_nuceat_tend, &inv_exner, &exner, &inv_cld_frac_l, &inv_cld_frac_i, &inv_cld_frac_r, &latent_heat_vapor, &latent_heat_sublim, &latent_heat_fusion, &nccn_prescribed, &T_atm, &rho, &inv_rho, &qv_sat_l, &qv_sat_i, &qv_supersat_i, &rhofacr, &rhofaci, @@ -727,26 +155,11 @@ P3MainPart1Data::P3MainPart1Data( do_predict_nc(do_predict_nc_), do_prescribed_CCN(do_prescribed_CCN_), dt(dt_) {} -void p3_main_part1(P3MainPart1Data& d) -{ - p3_init(); - p3_main_part1_c( - d.kts, d.kte, d.kbot, d.ktop, d.kdir, - d.do_predict_nc, d.do_prescribed_CCN, - d.dt, - d.pres, d.dpres, d.dz, d.nc_nuceat_tend, d.nccn_prescribed, d.inv_exner, d.exner, d.inv_cld_frac_l, d.inv_cld_frac_i, d.inv_cld_frac_r, d.latent_heat_vapor, - d.latent_heat_sublim, d.latent_heat_fusion, - d.T_atm, d.rho, d.inv_rho, d.qv_sat_l, d.qv_sat_i, d.qv_supersat_i, d.rhofacr, d.rhofaci, - d.acn, d.qv, d.th_atm, d.qc, d.nc, d.qr, d.nr, d.qi, d.ni, d.qm, d.bm, d.qc_incld, d.qr_incld, d.qi_incld, - d.qm_incld, d.nc_incld, d.nr_incld, d.ni_incld, d.bm_incld, - &d.is_nucleat_possible, &d.is_hydromet_present); -} - /////////////////////////////////////////////////////////////////////////////// P3MainPart2Data::P3MainPart2Data( Int kts_, Int kte_, Int kbot_, Int ktop_, Int kdir_, - bool do_predict_nc_, bool do_prescribed_CCN_, Real dt_) : + bool do_predict_nc_, bool do_prescribed_CCN_, Real dt_, Real, bool) : PhysicsTestData( { {(kte_ - kts_) + 1} }, { { &pres, &dpres, &dz, &nc_nuceat_tend, &inv_exner, &exner, &inv_cld_frac_l, &inv_cld_frac_i, &inv_cld_frac_r, &ni_activated, &inv_qc_relvar, &cld_frac_i, &cld_frac_l, &cld_frac_r, &qv_prev, &t_prev, &T_atm, &rho, &inv_rho, &qv_sat_l, &qv_sat_i, &qv_supersat_i, &rhofacr, &rhofaci, &acn, @@ -758,20 +171,6 @@ P3MainPart2Data::P3MainPart2Data( do_predict_nc(do_predict_nc_), do_prescribed_CCN(do_prescribed_CCN_), dt(dt_), inv_dt(1 / dt) {} -void p3_main_part2(P3MainPart2Data& d) -{ - p3_init(); - p3_main_part2_c( - d.kts, d.kte, d.kbot, d.ktop, d.kdir, d.do_predict_nc, d.do_prescribed_CCN, d.dt, d.inv_dt, - d.pres, d.inv_exner, d.inv_cld_frac_l, d.inv_cld_frac_i, d.inv_cld_frac_r, d.ni_activated, d.inv_qc_relvar, - d.cld_frac_i, d.cld_frac_l, d.cld_frac_r, d.qv_prev, d.t_prev, - d.T_atm, d.rho, d.inv_rho, d.qv_sat_l, d.qv_sat_i, d.qv_supersat_i, d.rhofaci, d.acn, d.qv, d.th_atm, d.qc, d.nc, d.qr, d.nr, d.qi, d.ni, - d.qm, d.bm, d.latent_heat_vapor, d.latent_heat_sublim, d.latent_heat_fusion, d.qc_incld, d.qr_incld, d.qi_incld, d.qm_incld, d.nc_incld, d.nr_incld, - d.ni_incld, d.bm_incld, d.mu_c, d.nu, d.lamc, d.cdist, d.cdist1, d.cdistr, d.mu_r, d.lamr, d.logn0r, d.qv2qi_depos_tend, d.precip_total_tend, - d.nevapr, d.qr_evap_tend, d.vap_liq_exchange, d.vap_ice_exchange, d.liq_ice_exchange, d.pratot, - d.prctot, &d.is_hydromet_present); -} - /////////////////////////////////////////////////////////////////////////////// P3MainPart3Data::P3MainPart3Data( @@ -787,21 +186,10 @@ P3MainPart3Data::P3MainPart3Data( kts(kts_), kte(kte_), kbot(kbot_), ktop(ktop_), kdir(kdir_) {} -void p3_main_part3(P3MainPart3Data& d) -{ - p3_init(); - p3_main_part3_c( - d.kts, d.kte, d.kbot, d.ktop, d.kdir, - d.inv_exner, d.cld_frac_l, d.cld_frac_r, d.cld_frac_i, - d.rho, d.inv_rho, d.rhofaci, d.qv, d.th_atm, d.qc, d.nc, d.qr, d.nr, d.qi, d.ni, d.qm, d.bm, d.latent_heat_vapor, d.latent_heat_sublim, - d.mu_c, d.nu, d.lamc, d.mu_r, d.lamr, d.vap_liq_exchange, - d. ze_rain, d.ze_ice, d.diag_vm_qi, d.diag_eff_radius_qi, d.diag_diam_qi, d.rho_qi, d.diag_equiv_reflectivity, d.diag_eff_radius_qc, d.diag_eff_radius_qr); -} - /////////////////////////////////////////////////////////////////////////////// P3MainData::P3MainData( - Int its_, Int ite_, Int kts_, Int kte_, Int it_, Real dt_, bool do_predict_nc_, bool do_prescribed_CCN_) : + Int its_, Int ite_, Int kts_, Int kte_, Int it_, Real dt_, bool do_predict_nc_, bool do_prescribed_CCN_, Real) : PhysicsTestData( { {(ite_ - its_) + 1, (kte_ - kts_) + 1}, {(ite_ - its_) + 1, (kte_ - kts_) + 2} }, { { &pres, &dz, &nc_nuceat_tend, &nccn_prescribed, &ni_activated, &dpres, &inv_exner, &cld_frac_i, &cld_frac_l, &cld_frac_r, &inv_qc_relvar, &qc, &nc, &qr, &nr, &qi, &qm, &ni, &bm, &qv, &th_atm, &qv_prev, &t_prev, @@ -812,51 +200,6 @@ P3MainData::P3MainData( its(its_), ite(ite_), kts(kts_), kte(kte_), it(it_), dt(dt_), do_predict_nc(do_predict_nc_), do_prescribed_CCN(do_prescribed_CCN_) {} -//This is the variable ordering from micro_p3.F90 -void p3_main(P3MainData& d) -{ - p3_init(); - d.transpose(); - p3_main_c( - d.qc, d.nc, d.qr, d.nr, d.th_atm, d.qv, d.dt, d.qi, d.qm, d.ni, - d.bm, d.pres, d.dz, d.nc_nuceat_tend, d.nccn_prescribed, d.ni_activated, d.inv_qc_relvar, d.it, d.precip_liq_surf, - d.precip_ice_surf, d.its, d.ite, d.kts, d.kte, d.diag_eff_radius_qc, d.diag_eff_radius_qi, d.diag_eff_radius_qr, - d.rho_qi, d.do_predict_nc, d.do_prescribed_CCN, d.dpres, d.inv_exner, d.qv2qi_depos_tend, - d.precip_liq_flux, d.precip_ice_flux, d.cld_frac_r, d.cld_frac_l, d.cld_frac_i, - d.liq_ice_exchange, d.vap_liq_exchange, d.vap_ice_exchange, d.qv_prev, d.t_prev, &d.elapsed_s); - d.transpose(); -} - -void ice_supersat_conservation(IceSupersatConservationData& d) -{ - p3_init(); - ice_supersat_conservation_c(&d.qidep, &d.qinuc, d.cld_frac_i, d.qv, d.qv_sat_i, d.latent_heat_sublim, d.t_atm, d.dt, d.qi2qv_sublim_tend, d.qr2qv_evap_tend); -} - -void nc_conservation(NcConservationData& d) -{ - p3_init(); - nc_conservation_c(d.nc, d.nc_selfcollect_tend, d.dt, &d.nc_collect_tend, &d.nc2ni_immers_freeze_tend, &d.nc_accret_tend, &d.nc2nr_autoconv_tend); -} - -void nr_conservation(NrConservationData& d) -{ - p3_init(); - nr_conservation_c(d.nr, d.ni2nr_melt_tend, d.nr_ice_shed_tend, d.ncshdc, d.nc2nr_autoconv_tend, d.dt, d.nmltratio, &d.nr_collect_tend, &d.nr2ni_immers_freeze_tend, &d.nr_selfcollect_tend, &d.nr_evap_tend); -} - -void ni_conservation(NiConservationData& d) -{ - p3_init(); - ni_conservation_c(d.ni, d.ni_nucleat_tend, d.nr2ni_immers_freeze_tend, d.nc2ni_immers_freeze_tend, d.dt, &d.ni2nr_melt_tend, &d.ni_sublim_tend, &d.ni_selfcollect_tend); -} - -void prevent_liq_supersaturation(PreventLiqSupersaturationData& d) -{ - p3_init(); - prevent_liq_supersaturation_c(d.pres, d.t_atm, d.qv, d.latent_heat_vapor, d.latent_heat_sublim, d.dt, d.qidep, d.qinuc, &d.qi2qv_sublim_tend, &d.qr2qv_evap_tend); -} - void IceSupersatConservationData::randomize(std::mt19937_64& engine) { std::uniform_real_distribution data_dist(0.0, 1.0); @@ -970,30 +313,10 @@ void PreventLiqSupersaturationData::randomize(std::mt19937_64& engine) */ } -// end _c impls - /////////////////////////////////////////////////////////////////////////////// -std::shared_ptr P3GlobalForFortran::s_views; - -const P3GlobalForFortran::Views& P3GlobalForFortran::get() -{ - if (!P3GlobalForFortran::s_views) { - P3GlobalForFortran::s_views = std::make_shared(); - P3F::init_kokkos_ice_lookup_tables(s_views->m_ice_table_vals, s_views->m_collect_table_vals); - P3F::init_kokkos_tables(s_views->m_vn_table_vals, s_views->m_vm_table_vals, - s_views->m_revap_table_vals, s_views->m_mu_r_table_vals, s_views->m_dnu); - } - return *P3GlobalForFortran::s_views; -} - -void P3GlobalForFortran::deinit() -{ - P3GlobalForFortran::s_views = nullptr; -} - // -// _f function definitions +// _host function definitions // template @@ -1006,7 +329,7 @@ std::vector ptr_to_arr(T** data, int n) } template -void calc_first_order_upwind_step_f_impl( +void calc_first_order_upwind_step_host_impl( Int kts, Int kte, Int kdir, Int kbot, Int k_qxtop, Real dt_sub, Real* rho, Real* inv_rho, Real* inv_dz, Real** fluxes, Real** vs, Real** qnx) @@ -1070,7 +393,7 @@ void calc_first_order_upwind_step_f_impl( } template -void generalized_sedimentation_f_impl( +void generalized_sedimentation_host_impl( Int kts, Int kte, Int kdir, Int k_qxtop, Int* k_qxbot, Int kbot, Real Co_max, Real* dt_left, Real* prt_accum, Real* inv_dz, Real* inv_rho, Real* rho, Real** vs, Real** fluxes, Real** qnx) @@ -1159,40 +482,40 @@ void generalized_sedimentation_f_impl( *k_qxbot = scalars[2] + 1; } -void calc_first_order_upwind_step_f( +void calc_first_order_upwind_step_host( Int kts, Int kte, Int kdir, Int kbot, Int k_qxtop, Real dt_sub, Real* rho, Real* inv_rho, Real* inv_dz, Int num_arrays, Real** fluxes, Real** vs, Real** qnx) { if (num_arrays == 1) { - calc_first_order_upwind_step_f_impl<1>(kts, kte, kdir, kbot, k_qxtop, dt_sub, rho, inv_rho, inv_dz, fluxes, vs, qnx); + calc_first_order_upwind_step_host_impl<1>(kts, kte, kdir, kbot, k_qxtop, dt_sub, rho, inv_rho, inv_dz, fluxes, vs, qnx); } else if (num_arrays == 2) { - calc_first_order_upwind_step_f_impl<2>(kts, kte, kdir, kbot, k_qxtop, dt_sub, rho, inv_rho, inv_dz, fluxes, vs, qnx); + calc_first_order_upwind_step_host_impl<2>(kts, kte, kdir, kbot, k_qxtop, dt_sub, rho, inv_rho, inv_dz, fluxes, vs, qnx); } else if (num_arrays == 4) { - calc_first_order_upwind_step_f_impl<4>(kts, kte, kdir, kbot, k_qxtop, dt_sub, rho, inv_rho, inv_dz, fluxes, vs, qnx); + calc_first_order_upwind_step_host_impl<4>(kts, kte, kdir, kbot, k_qxtop, dt_sub, rho, inv_rho, inv_dz, fluxes, vs, qnx); } else { EKAT_REQUIRE_MSG(false, "Unsupported num arrays in bridge calc_first_order_upwind_step_f: " << num_arrays); } } -void generalized_sedimentation_f( +void generalized_sedimentation_host( Int kts, Int kte, Int kdir, Int k_qxtop, Int* k_qxbot, Int kbot, Real Co_max, Real* dt_left, Real* prt_accum, Real* inv_dz, Real* inv_rho, Real* rho, Int num_arrays, Real** vs, Real** fluxes, Real** qnx) { if (num_arrays == 1) { - generalized_sedimentation_f_impl<1>(kts, kte, kdir, k_qxtop, k_qxbot, kbot, Co_max, dt_left, prt_accum, + generalized_sedimentation_host_impl<1>(kts, kte, kdir, k_qxtop, k_qxbot, kbot, Co_max, dt_left, prt_accum, inv_dz, inv_rho, rho, vs, fluxes, qnx); } else if (num_arrays == 2) { - generalized_sedimentation_f_impl<2>(kts, kte, kdir, k_qxtop, k_qxbot, kbot, Co_max, dt_left, prt_accum, + generalized_sedimentation_host_impl<2>(kts, kte, kdir, k_qxtop, k_qxbot, kbot, Co_max, dt_left, prt_accum, inv_dz, inv_rho, rho, vs, fluxes, qnx); } else if (num_arrays == 4) { - generalized_sedimentation_f_impl<4>(kts, kte, kdir, k_qxtop, k_qxbot, kbot, Co_max, dt_left, prt_accum, + generalized_sedimentation_host_impl<4>(kts, kte, kdir, k_qxtop, k_qxbot, kbot, Co_max, dt_left, prt_accum, inv_dz, inv_rho, rho, vs, fluxes, qnx); } else { @@ -1200,7 +523,7 @@ void generalized_sedimentation_f( } } -void cloud_sedimentation_f( +void cloud_sedimentation_host( Int kts, Int kte, Int ktop, Int kbot, Int kdir, Real* qc_incld, Real* rho, Real* inv_rho, Real* cld_frac_l, Real* acn, Real* inv_dz, Real dt, Real inv_dt, bool do_predict_nc, @@ -1226,7 +549,7 @@ void cloud_sedimentation_f( const Int nk_pack = ekat::npack(nk); // Set up views - const auto dnu = P3GlobalForFortran::dnu(); + const auto dnu = P3F::p3_init().dnu_table_vals; std::vector temp_d(CloudSedData::NUM_ARRAYS); @@ -1267,7 +590,7 @@ void cloud_sedimentation_f( ekat::device_to_host({qc, nc, nc_incld, mu_c, lamc, qc_tend, nc_tend}, nk, inout_views); } -void ice_sedimentation_f( +void ice_sedimentation_host( Int kts, Int kte, Int ktop, Int kbot, Int kdir, Real* rho, Real* inv_rho, Real* rhofaci, Real* cld_frac_i, Real* inv_dz, Real dt, Real inv_dt, @@ -1317,7 +640,7 @@ void ice_sedimentation_f( ni_tend_d (temp_d[14]); // Call core function from kernel - auto ice_table_vals = P3GlobalForFortran::ice_table_vals(); + auto ice_table_vals = P3F::p3_init().ice_table_vals; auto policy = ekat::ExeSpaceUtils::get_default_team_policy(1, nk_pack); ekat::WorkspaceManager wsm(rho_d.extent(0), 6, policy); Real my_precip_ice_surf = 0; @@ -1329,7 +652,7 @@ void ice_sedimentation_f( nk, ktop, kbot, kdir, dt, inv_dt, qi_d, qi_incld_d, ni_d, ni_incld_d, qm_d, qm_incld_d, bm_d, bm_incld_d, qi_tend_d, ni_tend_d, ice_table_vals, - precip_ice_surf_k, physics::P3_Constants()); + precip_ice_surf_k, P3F::P3Runtime()); }, my_precip_ice_surf); *precip_ice_surf += my_precip_ice_surf; @@ -1340,7 +663,7 @@ void ice_sedimentation_f( ekat::device_to_host({qi, qi_incld, ni, ni_incld, qm, qm_incld, bm, bm_incld, qi_tend, ni_tend}, nk, inout_views); } -void rain_sedimentation_f( +void rain_sedimentation_host( Int kts, Int kte, Int ktop, Int kbot, Int kdir, Real* qr_incld, Real* rho, Real* inv_rho, Real* rhofacr, Real* cld_frac_r, Real* inv_dz, Real dt, Real inv_dt, @@ -1390,8 +713,9 @@ void rain_sedimentation_f( precip_liq_flux_d(temp_d[13]); // Call core function from kernel - auto vn_table_vals = P3GlobalForFortran::vn_table_vals(); - auto vm_table_vals = P3GlobalForFortran::vm_table_vals(); + auto tables = P3F::p3_init(); + auto vn_table_vals = tables.vn_table_vals; + auto vm_table_vals = tables.vm_table_vals; auto policy = ekat::ExeSpaceUtils::get_default_team_policy(1, nk_pack); ekat::WorkspaceManager wsm(rho_d.extent(0), 4, policy); Real my_precip_liq_surf = 0; @@ -1402,7 +726,7 @@ void rain_sedimentation_f( team, wsm.get_workspace(team), vn_table_vals, vm_table_vals, nk, ktop, kbot, kdir, dt, inv_dt, qr_d, nr_d, nr_incld_d, mu_r_d, lamr_d, precip_liq_flux_d, qr_tend_d, nr_tend_d, - precip_liq_surf_k, physics::P3_Constants()); + precip_liq_surf_k, P3F::P3Runtime()); }, my_precip_liq_surf); *precip_liq_surf += my_precip_liq_surf; @@ -1415,7 +739,7 @@ void rain_sedimentation_f( ekat::device_to_host({qr, nr, nr_incld, mu_r, lamr, qr_tend, nr_tend, precip_liq_flux}, sizes_out, inout_views); } -void homogeneous_freezing_f( +void homogeneous_freezing_host( Int kts, Int kte, Int ktop, Int kbot, Int kdir, Real* T_atm, Real* inv_exner, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* th_atm) @@ -1476,7 +800,7 @@ void homogeneous_freezing_f( ekat::device_to_host({qc, nc, qr, nr, qi, ni, qm, bm, th_atm}, nk, inout_views); } -void check_values_f(Real* qv, Real* temp, Int kstart, Int kend, +void check_values_host(Real* qv, Real* temp, Int kstart, Int kend, Int timestepcount, bool force_abort, Int source_ind, Real* col_loc) { using P3F = Functions; @@ -1509,7 +833,7 @@ void check_values_f(Real* qv, Real* temp, Int kstart, Int kend, }); } -void p3_main_part1_f( +void p3_main_part1_host( Int kts, Int kte, Int kbot, Int ktop, Int kdir, bool do_predict_nc, bool do_prescribed_CCN, Real dt, @@ -1600,7 +924,7 @@ void p3_main_part1_f( t_d, rho_d, inv_rho_d, qv_sat_l_d, qv_sat_i_d, qv_supersat_i_d, rhofacr_d, rhofaci_d, acn_d, qv_d, th_atm_d, qc_d, nc_d, qr_d, nr_d, qi_d, ni_d, qm_d, bm_d, qc_incld_d, qr_incld_d, qi_incld_d, qm_incld_d, nc_incld_d, nr_incld_d, ni_incld_d, bm_incld_d, - bools_d(0), bools_d(1), physics::P3_Constants()); + bools_d(0), bools_d(1), P3F::P3Runtime()); }); // Sync back to host @@ -1621,7 +945,7 @@ void p3_main_part1_f( *is_hydromet_present = bools_h(1); } -void p3_main_part2_f( +void p3_main_part2_host( Int kts, Int kte, Int kbot, Int ktop, Int kdir, bool do_predict_nc, bool do_prescribed_CCN, Real dt, Real inv_dt, Real* pres, Real* dpres, Real* dz, Real* nc_nuceat_tend, Real* inv_exner, Real* exner, Real* inv_cld_frac_l, Real* inv_cld_frac_i, Real* inv_cld_frac_r, Real* ni_activated, Real* inv_qc_relvar, Real* cld_frac_i, Real* cld_frac_l, Real* cld_frac_r, Real* qv_prev, Real* t_prev, @@ -1730,10 +1054,11 @@ void p3_main_part2_f( t_prev_d (temp_d[current_index++]); // Call core function from kernel - const auto dnu = P3GlobalForFortran::dnu(); - const auto ice_table_vals = P3GlobalForFortran::ice_table_vals(); - const auto collect_table_vals = P3GlobalForFortran::collect_table_vals(); - const auto revap_table_vals = P3GlobalForFortran::revap_table_vals(); + auto tables = P3F::p3_init(); + const auto dnu = tables.dnu_table_vals; + const auto ice_table_vals = tables.ice_table_vals; + const auto collect_table_vals = tables.collect_table_vals; + const auto revap_table_vals = tables.revap_table_vals; bview_1d bools_d("bools", 1); auto policy = ekat::ExeSpaceUtils::get_default_team_policy(1, nk_pack); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { @@ -1748,7 +1073,7 @@ void p3_main_part2_f( qm_incld_d, nc_incld_d, nr_incld_d, ni_incld_d, bm_incld_d, mu_c_d, nu_d, lamc_d, cdist_d, cdist1_d, cdistr_d, mu_r_d, lamr_d, logn0r_d, qv2qi_depos_tend_d, precip_total_tend_d, nevapr_d, qr_evap_tend_d, vap_liq_exchange_d, - vap_ice_exchange_d, liq_ice_exchange_d, pratot_d, prctot_d, bools_d(0),nk, physics::P3_Constants()); + vap_ice_exchange_d, liq_ice_exchange_d, pratot_d, prctot_d, bools_d(0),nk, P3F::P3Runtime()); }); // Sync back to host. Skip intent in variables. @@ -1777,7 +1102,7 @@ void p3_main_part2_f( *is_hydromet_present = bools_h(0); } -void p3_main_part3_f( +void p3_main_part3_host( Int kts, Int kte, Int kbot, Int ktop, Int kdir, Real* inv_exner, Real* cld_frac_l, Real* cld_frac_r, Real* cld_frac_i, Real* rho, Real* inv_rho, Real* rhofaci, Real* qv, Real* th_atm, Real* qc, @@ -1853,8 +1178,9 @@ void p3_main_part3_f( diag_eff_radius_qr_d (temp_d[current_index++]); // Call core function from kernel - const auto dnu = P3GlobalForFortran::dnu(); - const auto ice_table_vals = P3GlobalForFortran::ice_table_vals(); + auto tables = P3F::p3_init(); + const auto dnu = tables.dnu_table_vals; + const auto ice_table_vals = tables.ice_table_vals; auto policy = ekat::ExeSpaceUtils::get_default_team_policy(1, nk_pack); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { @@ -1865,7 +1191,7 @@ void p3_main_part3_f( mu_c_d, nu_d, lamc_d, mu_r_d, lamr_d, vap_liq_exchange_d, ze_rain_d, ze_ice_d, diag_vm_qi_d, diag_eff_radius_qi_d, diag_diam_qi_d, rho_qi_d, - diag_equiv_reflectivity_d, diag_eff_radius_qc_d, diag_eff_radius_qr_d, physics::P3_Constants()); + diag_equiv_reflectivity_d, diag_eff_radius_qc_d, diag_eff_radius_qr_d, P3F::P3Runtime()); }); // Sync back to host @@ -1885,7 +1211,7 @@ void p3_main_part3_f( nk, inout_views); } -Int p3_main_f( +Int p3_main_host( Real* qc, Real* nc, Real* qr, Real* nr, Real* th_atm, Real* qv, Real dt, Real* qi, Real* qm, Real* ni, Real* bm, Real* pres, Real* dz, Real* nc_nuceat_tend, Real* nccn_prescribed, Real* ni_activated, Real* inv_qc_relvar, Int it, Real* precip_liq_surf, @@ -1902,12 +1228,6 @@ Int p3_main_f( using sview_1d = typename P3F::view_1d; using sview_2d = typename P3F::view_2d; - using view_1d_table = typename P3F::view_1d_table; - using view_2d_table = typename P3F::view_2d_table; - using view_ice_table = typename P3F::view_ice_table; - using view_collect_table = typename P3F::view_collect_table; - using view_dnu_table = typename P3F::view_dnu_table; - EKAT_REQUIRE_MSG(its == 1, "its must be 1, got " << its); EKAT_REQUIRE_MSG(kts == 1, "kts must be 1, got " << kts); @@ -1944,7 +1264,7 @@ Int p3_main_f( } } - ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d, true); + ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d); int counter = 0; view_2d @@ -2048,16 +1368,7 @@ Int p3_main_f( #endif // load tables - view_1d_table mu_r_table_vals; - view_2d_table vn_table_vals, vm_table_vals, revap_table_vals; - view_ice_table ice_table_vals; - view_collect_table collect_table_vals; - view_dnu_table dnu_table_vals; - P3F::init_kokkos_ice_lookup_tables(ice_table_vals, collect_table_vals); - P3F::init_kokkos_tables(vn_table_vals, vm_table_vals, revap_table_vals, mu_r_table_vals, dnu_table_vals); - - P3F::P3LookupTables lookup_tables{mu_r_table_vals, vn_table_vals, vm_table_vals, revap_table_vals, - ice_table_vals, collect_table_vals, dnu_table_vals}; + auto lookup_tables = P3F::p3_init(); P3F::P3Runtime runtime_options{740.0e3}; // Create local workspace @@ -2069,7 +1380,7 @@ Int p3_main_f( #ifdef SCREAM_P3_SMALL_KERNELS temporaries, #endif - workspace_mgr, nj, nk, physics::P3_Constants()); + workspace_mgr, nj, nk); Kokkos::parallel_for(nj, KOKKOS_LAMBDA(const Int& i) { precip_liq_surf_temp_d(0, i / Spack::n)[i % Spack::n] = precip_liq_surf_d(i); @@ -2097,7 +1408,7 @@ Int p3_main_f( rho_qi, qv2qi_depos_tend, liq_ice_exchange, vap_liq_exchange, vap_ice_exchange, precip_liq_flux, precip_ice_flux, precip_liq_surf, precip_ice_surf }, - dim1_sizes_out, dim2_sizes_out, inout_views, true); + dim1_sizes_out, dim2_sizes_out, inout_views); return elapsed_microsec; } diff --git a/components/eamxx/src/physics/p3/p3_functions_f90.hpp b/components/eamxx/src/physics/p3/tests/infra/p3_test_data.hpp similarity index 78% rename from components/eamxx/src/physics/p3/p3_functions_f90.hpp rename to components/eamxx/src/physics/p3/tests/infra/p3_test_data.hpp index d1c29d3de0de..775120e080dc 100644 --- a/components/eamxx/src/physics/p3/p3_functions_f90.hpp +++ b/components/eamxx/src/physics/p3/tests/infra/p3_test_data.hpp @@ -4,84 +4,21 @@ #include "physics/p3/p3_functions.hpp" #include "physics/share/physics_test_data.hpp" #include "share/scream_types.hpp" +#include "ekat/util/ekat_file_utils.hpp" #include #include #include // for shared_ptr -// -// Bridge functions to call fortran version of p3 functions from C++ -// - namespace scream { namespace p3 { -// -// Singleton for holding the same global data that are maintained in -// micro_p3, but for use in C++. This data is necessary to complete -// the "bridge" when calling C++ from micro_p3. -// -struct P3GlobalForFortran -{ - using P3F = Functions; - - using view_1d_table = typename P3F::view_1d_table; - using view_2d_table = typename P3F::view_2d_table; - using view_ice_table = typename P3F::view_ice_table; - using view_collect_table = typename P3F::view_collect_table; - using view_dnu_table = typename P3F::view_dnu_table; - - // All kokkos views must be destructed before Kokkos::finalize - static void deinit(); - - static const view_1d_table& mu_r_table_vals() { return get().m_mu_r_table_vals; } - static const view_2d_table& vn_table_vals() { return get().m_vn_table_vals; } - static const view_2d_table& vm_table_vals() { return get().m_vm_table_vals; } - static const view_2d_table& revap_table_vals() { return get().m_revap_table_vals; } - static const view_ice_table& ice_table_vals() { return get().m_ice_table_vals; } - static const view_collect_table& collect_table_vals() { return get().m_collect_table_vals; } - static const view_dnu_table& dnu() { return get().m_dnu; } - - P3GlobalForFortran() = delete; - ~P3GlobalForFortran() = delete; - P3GlobalForFortran(const P3GlobalForFortran&) = delete; - P3GlobalForFortran& operator=(const P3GlobalForFortran&) = delete; - - private: - struct Views { - view_1d_table m_mu_r_table_vals; - view_2d_table m_vn_table_vals, m_vm_table_vals, m_revap_table_vals; - view_ice_table m_ice_table_vals; - view_collect_table m_collect_table_vals; - view_dnu_table m_dnu; - }; - - static const Views& get(); - static std::shared_ptr s_views; -}; - /////////////////////////////////////////////////////////////////////////////// -struct P3InitAFortranData -{ - // Must use Host as device, f90 code might not be able to use Device memory - using P3F = Functions; - using P3C = typename P3F::P3C; - - using view_ice_table = typename P3F::KT::template lview; - using view_collect_table = typename P3F::KT::template lview; - - // Need to be LayoutLeft to be fortran compatible - view_ice_table ice_table_vals; - view_collect_table collect_table_vals; - - P3InitAFortranData() : - ice_table_vals("P3InitAFortranData::ice_table_vals"), - collect_table_vals("P3InitAFortranData::collect_table_vals") - {} -}; - -/////////////////////////////////////////////////////////////////////////////// +/** + * Structs for holding data related to specific P3 calls; these are used for + * the BFB unit tests. + */ struct LookupIceData { @@ -91,6 +28,8 @@ struct LookupIceData // Outputs Int dumi, dumjj, dumii, dumzz; Real dum1, dum4, dum5, dum6; + + PTD_RW_SCALARS_ONLY(8, dumi, dumjj, dumii, dumzz, dum1, dum4, dum5, dum6); }; /////////////////////////////////////////////////////////////////////////////// @@ -103,6 +42,8 @@ struct LookupIceDataB // Outputs Int dumj; Real dum3; + + PTD_RW_SCALARS_ONLY(2, dumj, dum3); }; /////////////////////////////////////////////////////////////////////////////// @@ -115,6 +56,8 @@ struct AccessLookupTableData // Outputs Real proc; + + PTD_RW_SCALARS_ONLY(1, proc); }; /////////////////////////////////////////////////////////////////////////////// @@ -128,6 +71,8 @@ struct AccessLookupTableCollData // Outputs Real proc; + + PTD_RW_SCALARS_ONLY(1, proc); }; /////////////////////////////////////////////////////////////////////////////// @@ -145,6 +90,11 @@ struct BackToCellAverageData // This populates all fields with test data within [0,1]. void randomize(std::mt19937_64& engine); + + PTD_RW_SCALARS_ONLY(31, qc2qr_accret_tend, qr2qv_evap_tend, qc2qr_autoconv_tend, nc_accret_tend, nc_selfcollect_tend, nc2nr_autoconv_tend, nr_selfcollect_tend, nr_evap_tend, ncautr, qcnuc, + nc_nuceat_tend, qi2qv_sublim_tend, nr_ice_shed_tend, qc2qi_hetero_freeze_tend, qr2qi_collect_tend, qc2qr_ice_shed_tend, qi2qr_melt_tend, qc2qi_collect_tend, qr2qi_immers_freeze_tend, ni2nr_melt_tend, + nc_collect_tend, ncshdc, nc2ni_immers_freeze_tend, nr_collect_tend, ni_selfcollect_tend, qv2qi_vapdep_tend, nr2ni_immers_freeze_tend, ni_sublim_tend, qv2qi_nucleat_tend, ni_nucleat_tend, + qc2qi_berg_tend); }; /////////////////////////////////////////////////////////////////////////////// @@ -156,6 +106,8 @@ struct CloudWaterConservationData //output Real qc2qr_autoconv_tend, qc2qr_accret_tend, qc2qi_collect_tend, qc2qi_hetero_freeze_tend, qc2qr_ice_shed_tend, qc2qi_berg_tend, qi2qv_sublim_tend, qv2qi_vapdep_tend; + + PTD_RW_SCALARS_ONLY(8, qc2qr_autoconv_tend, qc2qr_accret_tend, qc2qi_collect_tend, qc2qi_hetero_freeze_tend, qc2qr_ice_shed_tend, qc2qi_berg_tend, qi2qv_sublim_tend, qv2qi_vapdep_tend); }; struct RainWaterConservationData @@ -165,6 +117,8 @@ struct RainWaterConservationData //output Real qr2qv_evap_tend, qr2qi_collect_tend, qr2qi_immers_freeze_tend; + + PTD_RW_SCALARS_ONLY(3, qr2qv_evap_tend, qr2qi_collect_tend, qr2qi_immers_freeze_tend); }; struct IceWaterConservationData @@ -174,6 +128,8 @@ struct IceWaterConservationData //output Real qi2qv_sublim_tend, qi2qr_melt_tend; + + PTD_RW_SCALARS_ONLY(2, qi2qv_sublim_tend, qi2qr_melt_tend); }; /////////////////////////////////////////////////////////////////////////////// @@ -185,6 +141,8 @@ struct CalcRimeDensityData // output Real vtrmi1, rho_qm_cloud; + + PTD_RW_SCALARS_ONLY(2, vtrmi1, rho_qm_cloud); }; /////////////////////////////////////////////////////////////////////////////// @@ -196,6 +154,8 @@ struct CldliqImmersionFreezingData // output Real qc2qi_hetero_freeze_tend, nc2ni_immers_freeze_tend; + + PTD_RW_SCALARS_ONLY(2, qc2qi_hetero_freeze_tend, nc2ni_immers_freeze_tend); }; /////////////////////////////////////////////////////////////////////////////// @@ -207,6 +167,8 @@ struct RainImmersionFreezingData // output Real qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend; + + PTD_RW_SCALARS_ONLY(2, qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend); }; /////////////////////////////////////////////////////////////////////////////// @@ -218,6 +180,8 @@ struct DropletSelfCollectionData // output Real nc_selfcollect_tend; + + PTD_RW_SCALARS_ONLY(1, nc_selfcollect_tend); }; /////////////////////////////////////////////////////////////////////////////// @@ -229,6 +193,8 @@ struct CloudRainAccretionData // output Real qc2qr_accret_tend, nc_accret_tend; + + PTD_RW_SCALARS_ONLY(2, qc2qr_accret_tend, nc_accret_tend); }; /////////////////////////////////////////////////////////////////////////////// @@ -236,15 +202,12 @@ struct CloudRainAccretionData struct CloudWaterAutoconversionData { // inputs - Real rho; - Real qc_incld; - Real nc_incld; - Real inv_qc_relvar; + Real rho, qc_incld, nc_incld, inv_qc_relvar; // output - Real qc2qr_autoconv_tend; - Real nc2nr_autoconv_tend; - Real ncautr; + Real qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr; + + PTD_RW_SCALARS_ONLY(3, qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr); }; /////////////////////////////////////////////////////////////////////////////// @@ -256,6 +219,8 @@ struct RainSelfCollectionData //output Real nr_selfcollect_tend; + + PTD_RW_SCALARS_ONLY(1, nr_selfcollect_tend); }; /////////////////////////////////////////////////////////////////////////////// @@ -266,6 +231,8 @@ struct ImposeMaxTotalNiData{ //input Real max_total_ni, inv_rho_local; + + PTD_RW_SCALARS_ONLY(2, ni_local, inv_rho_local); }; /////////////////////////////////////////////////////////////////////////////// @@ -277,6 +244,8 @@ struct IceMeltingData // output Real qi2qr_melt_tend,ni2nr_melt_tend; + + PTD_RW_SCALARS_ONLY(2, qi2qr_melt_tend, ni2nr_melt_tend); }; /////////////////////////////////////////////////////////////////////////////// @@ -297,6 +266,8 @@ struct GetCloudDsd2Data // Outputs Real nc_out, mu_c, nu, lamc, cdist, cdist1; + + PTD_RW_SCALARS_ONLY(6, nc_out, mu_c, nu, lamc, cdist, cdist1) }; ////////////////////////////////////////////////////////////////////////// @@ -308,6 +279,8 @@ struct GetRainDsd2Data // Outputs Real nr_out, lamr, mu_r, cdistr, logn0r; + + PTD_RW_SCALARS_ONLY(5, nr_out, lamr, mu_r, cdistr, logn0r); }; /////////////////////////////////////////////////////////////////////////////// @@ -351,6 +324,8 @@ struct GenSedData : public CalcUpwindData PTD_DATA_COPY_CTOR(GenSedData, 10); PTD_ASSIGN_OP(GenSedData, 11, kts, kte, kdir, kbot, k_qxtop, num_arrays, dt_sub, Co_max, k_qxbot, dt_left, prt_accum); + PTD_RW(); + PTD_RW_SCALARS(11, kts, kte, kdir, kbot, k_qxtop, num_arrays, dt_sub, Co_max, k_qxbot, dt_left, prt_accum); }; /////////////////////////////////////////////////////////////////////////////// @@ -436,6 +411,8 @@ struct CalcBulkRhoRimeData // Outputs Real rho_rime; + + PTD_RW_SCALARS_ONLY(3, qi_rim, bi_rim, rho_rime); }; /////////////////////////////////////////////////////////////////////////////// @@ -470,6 +447,8 @@ struct ComputeRainFallVelocityData // Outputs Real mu_r, lamr, V_qr, V_nr; + + PTD_RW_SCALARS_ONLY(5, nr_incld, mu_r, lamr, V_qr, V_nr); }; /////////////////////////////////////////////////////////////////////////////// @@ -481,6 +460,8 @@ struct GetTimeSpacePhysVarsData //Outs Real mu, dv, sc, dqsdt, dqsidt, ab, abi, kap, eii; + + PTD_RW_SCALARS_ONLY(9, mu, dv, sc, dqsdt, dqsidt, ab, abi, kap, eii); }; /////////////////////////////////////////////////////////////////////////////// @@ -495,6 +476,8 @@ struct P3UpdatePrognosticIceData // In/outs Real th_atm, qv, qi, ni, qm, bm, qc, nc, qr, nr; + + PTD_RW_SCALARS_ONLY(10, th_atm, qv, qi, ni, qm, bm, qc, nc, qr, nr); }; /////////////////////////////////////////////////////////////////////////////// @@ -507,6 +490,8 @@ struct EvapRainData //Outs Real qr2qv_evap_tend, nr_evap_tend; + + PTD_RW_SCALARS_ONLY(2, qr2qv_evap_tend, nr_evap_tend); }; /////////////////////////////////////////////////////////////////////////////// @@ -522,6 +507,8 @@ struct P3UpdatePrognosticLiqData // In/outs Real th_atm, qv, qc, nc, qr, nr; + + PTD_RW_SCALARS_ONLY(6, th_atm, qv, qc, nc, qr, nr); }; /////////////////////////////////////////////////////////////////////////////// @@ -537,6 +524,7 @@ struct IceDepositionSublimationData // This populates all input fields with test data within [0,1]. void randomize(std::mt19937_64& engine); + PTD_RW_SCALARS_ONLY(4, qv2qi_vapdep_tend, qi2qv_sublim_tend, ni_sublim_tend, qc2qi_berg_tend); }; struct IceCldliqCollectionData @@ -548,6 +536,7 @@ struct IceCldliqCollectionData // Outputs Real qc2qi_collect_tend, nc_collect_tend, qc2qr_ice_shed_tend, ncshdc; + PTD_RW_SCALARS_ONLY(4, qc2qi_collect_tend, nc_collect_tend, qc2qr_ice_shed_tend, ncshdc); }; struct IceRainCollectionData @@ -559,6 +548,7 @@ struct IceRainCollectionData // Outputs Real qr2qi_collect_tend, nr_collect_tend; + PTD_RW_SCALARS_ONLY(2, qr2qi_collect_tend, nr_collect_tend); }; struct IceSelfCollectionData @@ -570,6 +560,7 @@ struct IceSelfCollectionData // Outputs Real ni_selfcollect_tend; + PTD_RW_SCALARS_ONLY(1, ni_selfcollect_tend); }; struct IceRelaxationData @@ -579,6 +570,8 @@ struct IceRelaxationData // Outputs Real epsi, epsi_tot; + + PTD_RW_SCALARS_ONLY(2, epsi, epsi_tot); }; struct CalcLiqRelaxationData @@ -591,6 +584,8 @@ struct CalcLiqRelaxationData // This populates all input fields with test data within [0,1]. void randomize(std::mt19937_64& engine); + + PTD_RW_SCALARS_ONLY(2, epsr, epsc); }; struct IceNucleationData @@ -602,6 +597,8 @@ struct IceNucleationData // Outputs Real qv2qi_nucleat_tend, ni_nucleat_tend; + + PTD_RW_SCALARS_ONLY(2, qv2qi_nucleat_tend, ni_nucleat_tend); }; struct IceWetGrowthData @@ -614,21 +611,8 @@ struct IceWetGrowthData bool log_wetgrowth; Real qr2qi_collect_tend, qc2qi_collect_tend, qc_growth_rate, nr_ice_shed_tend, qc2qr_ice_shed_tend; -}; -struct LatentHeatData : public PhysicsTestData -{ - static constexpr size_t NUM_ARRAYS = 3; - - // Inputs - Int its, ite, kts, kte; - - // Outputs - Real* v, *s, *f; - - LatentHeatData(Int its_, Int ite_, Int kts_, Int kte_); - - PTD_STD_DEF(LatentHeatData, 4, its, ite, kts, kte); + PTD_RW_SCALARS_ONLY(6, log_wetgrowth, qr2qi_collect_tend, qc2qi_collect_tend, qc_growth_rate, nr_ice_shed_tend, qc2qr_ice_shed_tend); }; struct CheckValuesData : public PhysicsTestData @@ -658,6 +642,8 @@ struct IncloudMixingData // Outputs Real qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, ni_incld, bm_incld; + + PTD_RW_SCALARS_ONLY(8, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, ni_incld, bm_incld); }; /////////////////////////////////////////////////////////////////////////////// @@ -682,9 +668,9 @@ struct P3MainPart1Data : public PhysicsTestData bool is_nucleat_possible, is_hydromet_present; P3MainPart1Data(Int kts_, Int kte_, Int kbot_, Int ktop_, Int kdir_, - bool do_predict_nc_, bool do_prescribed_CCN_, Real dt_); + bool do_predict_nc_, bool do_prescribed_CCN_, Real dt_, bool=false, bool=false); - PTD_STD_DEF(P3MainPart1Data, 8, kts, kte, kbot, ktop, kdir, do_predict_nc, do_prescribed_CCN, dt); + PTD_STD_DEF(P3MainPart1Data, 10, kts, kte, kbot, ktop, kdir, do_predict_nc, do_prescribed_CCN, dt, is_nucleat_possible, is_hydromet_present); Int nk() const { return (kte - kts) + 1; } }; @@ -711,10 +697,9 @@ struct P3MainPart2Data : public PhysicsTestData bool is_hydromet_present; P3MainPart2Data(Int kts_, Int kte_, Int kbot_, Int ktop_, Int kdir_, - bool do_predict_nc_, bool do_prescribed_CCN, Real dt_); + bool do_predict_nc_, bool do_prescribed_CCN, Real dt_, Real=0., bool=false); - PTD_DATA_COPY_CTOR(P3MainPart2Data, 8); - PTD_ASSIGN_OP(P3MainPart2Data, 10, kts, kte, kbot, ktop, kdir, do_predict_nc, do_prescribed_CCN, dt, inv_dt, is_hydromet_present); + PTD_STD_DEF(P3MainPart2Data, 10, kts, kte, kbot, ktop, kdir, do_predict_nc, do_prescribed_CCN, dt, inv_dt, is_hydromet_present); Int nk() const { return (kte - kts) + 1; } }; @@ -766,9 +751,9 @@ struct P3MainData : public PhysicsTestData *precip_liq_flux, *precip_ice_flux, *precip_liq_surf, *precip_ice_surf; Real elapsed_s; - P3MainData(Int its_, Int ite_, Int kts_, Int kte_, Int it_, Real dt_, bool do_predict_nc_, bool do_prescribed_CCN_); + P3MainData(Int its_, Int ite_, Int kts_, Int kte_, Int it_, Real dt_, bool do_predict_nc_, bool do_prescribed_CCN_, Real=0.); - PTD_STD_DEF(P3MainData, 8, its, ite, kts, kte, it, dt, do_predict_nc, do_prescribed_CCN); + PTD_STD_DEF(P3MainData, 9, its, ite, kts, kte, it, dt, do_predict_nc, do_prescribed_CCN, elapsed_s); }; struct IceSupersatConservationData { @@ -779,6 +764,8 @@ struct IceSupersatConservationData { Real qidep, qinuc; void randomize(std::mt19937_64& engine); + + PTD_RW_SCALARS_ONLY(2, qidep, qinuc); }; struct NcConservationData { @@ -789,6 +776,8 @@ struct NcConservationData { Real nc_collect_tend, nc2ni_immers_freeze_tend, nc_accret_tend, nc2nr_autoconv_tend; void randomize(std::mt19937_64& engine); + + PTD_RW_SCALARS_ONLY(4, nc_collect_tend, nc2ni_immers_freeze_tend, nc_accret_tend, nc2nr_autoconv_tend); }; struct NrConservationData { @@ -799,6 +788,8 @@ struct NrConservationData { Real nr_collect_tend, nr2ni_immers_freeze_tend, nr_selfcollect_tend, nr_evap_tend; void randomize(std::mt19937_64& engine); + + PTD_RW_SCALARS_ONLY(4, nr_collect_tend, nr2ni_immers_freeze_tend, nr_selfcollect_tend, nr_evap_tend); }; struct NiConservationData { @@ -809,6 +800,8 @@ struct NiConservationData { Real ni2nr_melt_tend, ni_sublim_tend, ni_selfcollect_tend; void randomize(std::mt19937_64& engine); + + PTD_RW_SCALARS_ONLY(3, ni2nr_melt_tend, ni_sublim_tend, ni_selfcollect_tend); }; struct PreventLiqSupersaturationData { @@ -820,101 +813,52 @@ struct PreventLiqSupersaturationData { // This populates all fields with test data within [0,1]. void randomize(std::mt19937_64& engine); + + PTD_RW_SCALARS_ONLY(2, qi2qv_sublim_tend, qr2qv_evap_tend); }; -// Glue functions to call fortran from from C++ with the Data struct -void p3_init_a(P3InitAFortranData& d); -void find_lookuptable_indices_1a(LookupIceData& d); -void find_lookuptable_indices_1b(LookupIceDataB& d); -void access_lookup_table(AccessLookupTableData& d); -void access_lookup_table_coll(AccessLookupTableCollData& d); -void back_to_cell_average(BackToCellAverageData& d); -void cloud_water_conservation(CloudWaterConservationData& d); -void rain_water_conservation(RainWaterConservationData& d); -void ice_water_conservation(IceWaterConservationData& d); -void calc_rime_density(CalcRimeDensityData& d); -void cldliq_immersion_freezing(CldliqImmersionFreezingData& d); -void rain_immersion_freezing(RainImmersionFreezingData& d); -void droplet_self_collection(DropletSelfCollectionData& d); -void cloud_rain_accretion(CloudRainAccretionData& d); -void cloud_water_autoconversion(CloudWaterAutoconversionData& d); -void rain_self_collection(RainSelfCollectionData& d); -void impose_max_total_ni(ImposeMaxTotalNiData& d); -void ice_melting(IceMeltingData& d); -Real subgrid_variance_scaling(SubgridVarianceScalingData& d); -void get_cloud_dsd2(GetCloudDsd2Data& d); -void get_rain_dsd2(GetRainDsd2Data& d); -void calc_first_order_upwind_step(CalcUpwindData& d); -void generalized_sedimentation(GenSedData& d); -void cloud_sedimentation(CloudSedData& d); -void ice_sedimentation(IceSedData& d); -void rain_sedimentation(RainSedData& d); -void calc_bulk_rho_rime(CalcBulkRhoRimeData& d); -void homogeneous_freezing(HomogeneousFreezingData& d); -void compute_rain_fall_velocity(ComputeRainFallVelocityData& d); -void get_time_space_phys_variables(GetTimeSpacePhysVarsData& d); -void update_prognostic_ice(P3UpdatePrognosticIceData& d); -void evaporate_rain(EvapRainData& d); -void update_prognostic_liquid(P3UpdatePrognosticLiqData& d); -void ice_deposition_sublimation(IceDepositionSublimationData& d); -void ice_cldliq_collection(IceCldliqCollectionData& d); -void ice_rain_collection(IceRainCollectionData& d); -void ice_self_collection(IceSelfCollectionData& d); -void ice_relaxation_timescale(IceRelaxationData& d); -void calc_liq_relaxation_timescale(CalcLiqRelaxationData& d); -void ice_nucleation(IceNucleationData& d); -void ice_cldliq_wet_growth(IceWetGrowthData& d); -void get_latent_heat(LatentHeatData& d); -void check_values(CheckValuesData& d); -void calculate_incloud_mixingratios(IncloudMixingData& d); -void p3_main_part1(P3MainPart1Data& d); -void p3_main_part2(P3MainPart2Data& d); -void p3_main_part3(P3MainPart3Data& d); -void p3_main(P3MainData& d); - -void ice_supersat_conservation(IceSupersatConservationData& d); -void nc_conservation(NcConservationData& d); -void nr_conservation(NrConservationData& d); -void ni_conservation(NiConservationData& d); -void prevent_liq_supersaturation(PreventLiqSupersaturationData& d); -extern "C" { // _f function decls - -void calc_first_order_upwind_step_f( +/** + * Convenience functions for calling p3 routines from the host with scalar data. + * These function will pack your data, sync it to device, call the p3 function, + * then sync back to host and unpack. These are used by the BFB unit tests. + */ + +void calc_first_order_upwind_step_host( Int kts, Int kte, Int kdir, Int kbot, Int k_qxtop, Real dt_sub, Real* rho, Real* inv_rho, Real* inv_dz, Int num_arrays, Real** fluxes, Real** vs, Real** qnx); -void generalized_sedimentation_f(Int kts, Int kte, Int kdir, Int k_qxtop, Int *k_qxbot, Int kbot, Real Co_max, +void generalized_sedimentation_host(Int kts, Int kte, Int kdir, Int k_qxtop, Int *k_qxbot, Int kbot, Real Co_max, Real* dt_left, Real* prt_accum, Real* inv_dz, Real* inv_rho, Real* rho, Int num_arrays, Real** vs, Real** fluxes, Real** qnx); -void cloud_sedimentation_f( +void cloud_sedimentation_host( Int kts, Int kte, Int ktop, Int kbot, Int kdir, Real* qc_incld, Real* rho, Real* inv_rho, Real* cld_frac_l, Real* acn, Real* inv_dz, Real dt, Real inv_dt, bool do_predict_nc, Real* qc, Real* nc, Real* nc_incld, Real* mu_c, Real* lamc, Real* precip_liq_surf, Real* qc_tend, Real* nc_tend); -void ice_sedimentation_f( +void ice_sedimentation_host( Int kts, Int kte, Int ktop, Int kbot, Int kdir, Real* rho, Real* inv_rho, Real* rhofaci, Real* cld_frac_i, Real* inv_dz, Real dt, Real inv_dt, Real* qi, Real* qi_incld, Real* ni, Real* qm, Real* qm_incld, Real* bm, Real* bm_incld, Real* ni_incld, Real* precip_ice_surf, Real* qi_tend, Real* ni_tend); -void rain_sedimentation_f( +void rain_sedimentation_host( Int kts, Int kte, Int ktop, Int kbot, Int kdir, Real* qr_incld, Real* rho, Real* inv_rho, Real* rhofacr, Real* cld_frac_r, Real* inv_dz, Real dt, Real inv_dt, Real* qr, Real* nr, Real* nr_incld, Real* mu_r, Real* lamr, Real* precip_liq_surf, Real* precip_liq_flux, Real* qr_tend, Real* nr_tend); -void homogeneous_freezing_f( +void homogeneous_freezing_host( Int kts, Int kte, Int ktop, Int kbot, Int kdir, Real* T_atm, Real* inv_exner, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* th_atm); -void check_values_f(Real* Qv, Real* temp, Int kstart, Int kend, - Int timestepcount, bool force_abort, Int source_ind, Real* col_loc); +void check_values_host(Real* Qv, Real* temp, Int kstart, Int kend, + Int timestepcount, bool force_abort, Int source_ind, Real* col_loc); -void p3_main_part1_f( +void p3_main_part1_host( Int kts, Int kte, Int kbot, Int ktop, Int kdir, bool do_predict_nc, bool do_prescribed_CCN, Real dt, @@ -925,7 +869,7 @@ void p3_main_part1_f( Real* qm_incld, Real* nc_incld, Real* nr_incld, Real* ni_incld, Real* bm_incld, bool* is_nucleat_possible, bool* is_hydromet_present); -void p3_main_part2_f( +void p3_main_part2_host( Int kts, Int kte, Int kbot, Int ktop, Int kdir, bool do_predict_nc, bool do_prescribed_CCN, Real dt, Real inv_dt, Real* pres, Real* dpres, Real* dz, Real* nc_nuceat_tend, Real* inv_exner, Real* exner, Real* inv_cld_frac_l, Real* inv_cld_frac_i, Real* inv_cld_frac_r, Real* ni_activated, Real* inv_qc_relvar, Real* cld_frac_i, Real* cld_frac_l, Real* cld_frac_r, Real* qv_prev, Real* t_prev, Real* T_atm, Real* rho, Real* inv_rho, Real* qv_sat_l, Real* qv_sat_i, Real* qv_supersat_i, Real* rhofacr, Real* rhofaci, Real* acn, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, @@ -934,14 +878,14 @@ void p3_main_part2_f( Real* nevapr, Real* qr_evap_tend, Real* vap_liq_exchange, Real* vap_ice_exchange, Real* liq_ice_exchange, Real* pratot, Real* prctot, bool* is_hydromet_present); -void p3_main_part3_f( +void p3_main_part3_host( Int kts, Int kte, Int kbot, Int ktop, Int kdir, Real* inv_exner, Real* cld_frac_l, Real* cld_frac_r, Real* cld_frac_i, Real* rho, Real* inv_rho, Real* rhofaci, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* mu_c, Real* nu, Real* lamc, Real* mu_r, Real* lamr, Real* vap_liq_exchange, Real* ze_rain, Real* ze_ice, Real* diag_vm_qi, Real* diag_eff_radius_qi, Real* diag_diam_qi, Real* rho_qi, Real* diag_equiv_reflectivity, Real* diag_eff_radius_qc, Real* diag_eff_radius_qr); -Int p3_main_f( +Int p3_main_host( Real* qc, Real* nc, Real* qr, Real* nr, Real* th_atm, Real* qv, Real dt, Real* qi, Real* qm, Real* ni, Real* bm, Real* pres, Real* dz, Real* nc_nuceat_tend, Real* nccn_prescribed, Real* ni_activated, Real* inv_qc_relvar, Int it, Real* precip_liq_surf, @@ -950,8 +894,6 @@ Int p3_main_f( Real* qv2qi_depos_tend, Real* precip_liq_flux, Real* precip_ice_flux, Real* cld_frac_r, Real* cld_frac_l, Real* cld_frac_i, Real* liq_ice_exchange, Real* vap_liq_exchange, Real* vap_ice_exchange, Real* qv_prev, Real* t_prev); -} // end _f function decls - } // namespace p3 } // namespace scream diff --git a/components/eamxx/src/physics/p3/tests/p3_unit_tests_common.hpp b/components/eamxx/src/physics/p3/tests/infra/p3_unit_tests_common.hpp similarity index 64% rename from components/eamxx/src/physics/p3/tests/p3_unit_tests_common.hpp rename to components/eamxx/src/physics/p3/tests/infra/p3_unit_tests_common.hpp index 68f4491f789d..232963ca7809 100644 --- a/components/eamxx/src/physics/p3/tests/p3_unit_tests_common.hpp +++ b/components/eamxx/src/physics/p3/tests/infra/p3_unit_tests_common.hpp @@ -2,7 +2,14 @@ #define P3_UNIT_TESTS_COMMON_HPP #include "share/scream_types.hpp" +#include "share/util/scream_setup_random_test.hpp" #include "p3_functions.hpp" +#include "p3_data.hpp" +#include "ekat/util/ekat_test_utils.hpp" +#include "p3_test_data.hpp" + +#include +#include namespace scream { namespace p3 { @@ -20,6 +27,12 @@ namespace unit_test { struct UnitWrap { + enum BASELINE_ACTION { + NONE, + COMPARE, + GENERATE + }; + template struct UnitTest : public KokkosTypes { @@ -58,6 +71,66 @@ struct UnitWrap { static constexpr Int max_pack_size = 16; static constexpr Int num_test_itrs = max_pack_size / Spack::n; + struct Base { + std::string m_baseline_path; + std::string m_test_name; + BASELINE_ACTION m_baseline_action; + ekat::FILEPtr m_fid; + + Base() : + m_baseline_path(""), + m_test_name(Catch::getResultCapture().getCurrentTestName()), + m_baseline_action(NONE), + m_fid() + { + Functions::p3_init(); // many tests will need table data + auto& ts = ekat::TestSession::get(); + if (ts.flags["c"]) { + m_baseline_action = COMPARE; + } + else if (ts.flags["g"]) { + m_baseline_action = GENERATE; + } + else if (ts.flags["n"]) { + m_baseline_action = NONE; + } + m_baseline_path = ts.params["b"]; + + EKAT_REQUIRE_MSG( !(m_baseline_action != NONE && m_baseline_path == ""), + "P3 unit test flags problem: baseline actions were requested but no baseline path was provided"); + + std::string baseline_name = m_baseline_path + "/" + m_test_name; + if (m_baseline_action == COMPARE) { + m_fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + } + else if (m_baseline_action == GENERATE) { + m_fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + } + } + + ~Base() {} + + std::mt19937_64 get_engine() + { + if (m_baseline_action != COMPARE) { + // We can use any seed + int seed; + auto engine = setup_random_test(nullptr, &seed); + if (m_baseline_action == GENERATE) { + // Write the seed + ekat::write(&seed, 1, m_fid); + } + return engine; + } + else { + // Read the seed + int seed; + ekat::read(&seed, 1, m_fid); + return setup_random_test(seed); + } + } + }; + // Put struct decls here struct TestTableIce; struct TestTable3; @@ -102,7 +175,6 @@ struct UnitWrap { struct TestIceDepositionSublimation; struct TestPreventLiqSupersaturation; }; - }; } // namespace unit_test diff --git a/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp index aa9606e653e5..be08e5b0d4f3 100644 --- a/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -19,10 +19,10 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestP3CloudWaterAutoconversion +struct UnitWrap::UnitTest::TestP3CloudWaterAutoconversion : public UnitWrap::UnitTest::Base { -static void cloud_water_autoconversion_unit_bfb_tests(){ +void cloud_water_autoconversion_unit_bfb_tests() { CloudWaterAutoconversionData cwadc[max_pack_size] = { // rho, qc_incld, nc_incld, inv_qc_relvar @@ -59,12 +59,14 @@ static void cloud_water_autoconversion_unit_bfb_tests(){ std::copy(&cwadc[0], &cwadc[0] + max_pack_size, cwadc_host.data()); Kokkos::deep_copy(cwadc_device, cwadc_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - cloud_water_autoconversion(cwadc[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + cwadc[i].read(Base::m_fid); + } } - // Run the lookup from a kernel and copy results back to host + // Run the lookup from a kernel and copy results back to host Kokkos::parallel_for(num_test_itrs, KOKKOS_LAMBDA(const Int& i) { const Int offset = i * Spack::n; @@ -80,8 +82,10 @@ static void cloud_water_autoconversion_unit_bfb_tests(){ ncautr[s] = cwadc_device(vs).ncautr; } - Functions::cloud_water_autoconversion(rho, qc_incld, nc_incld, - inv_qc_relvar, qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr, physics::P3_Constants()); + Functions::cloud_water_autoconversion( + rho, qc_incld, nc_incld, inv_qc_relvar, qc2qr_autoconv_tend, + nc2nr_autoconv_tend, ncautr, + p3::Functions::P3Runtime()); // Copy results back into views for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { @@ -99,7 +103,7 @@ static void cloud_water_autoconversion_unit_bfb_tests(){ Kokkos::deep_copy(cwadc_host, cwadc_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(cwadc[s].rho == cwadc_host(s).rho); REQUIRE(cwadc[s].qc_incld == cwadc_host(s).qc_incld); @@ -110,25 +114,34 @@ static void cloud_water_autoconversion_unit_bfb_tests(){ REQUIRE(cwadc[s].ncautr == cwadc_host(s).ncautr); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + cwadc_host(s).write(Base::m_fid); + } + } } - static void run_bfb(){ + void run_bfb() { cloud_water_autoconversion_unit_bfb_tests(); } - KOKKOS_FUNCTION static void autoconversion_is_positive(const Int &i, Int &errors){ + KOKKOS_FUNCTION static void autoconversion_is_positive(const Int &i, Int &errors){ const Spack rho(1.0), inv_qc_relvar(1.0); Spack qc_incld, nc_incld(1e7), qc2qr_autoconv_tend(0.0), nc2nr_autoconv_tend(0.0), ncautr(0.0); for(int si=0; si()); - if((qc2qr_autoconv_tend < 0.0).any()){errors++;} } + Functions::cloud_water_autoconversion( + rho, qc_incld, nc_incld, inv_qc_relvar, qc2qr_autoconv_tend, + nc2nr_autoconv_tend, ncautr, + p3::Functions::P3Runtime()); + if((qc2qr_autoconv_tend < 0.0).any()) { + errors++; + } + } - static void run_physics(){ + void run_physics(){ int nerr = 0; @@ -147,12 +160,14 @@ static void cloud_water_autoconversion_unit_bfb_tests(){ } // namespace p3 } // namespace scream -namespace{ +namespace { TEST_CASE("p3_cloud_water_autoconversion_test", "[p3_cloud_water_autoconversion_test]"){ - scream::p3::unit_test::UnitWrap::UnitTest::TestP3CloudWaterAutoconversion::run_physics(); - scream::p3::unit_test::UnitWrap::UnitTest::TestP3CloudWaterAutoconversion::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestP3CloudWaterAutoconversion; + + T t; + t.run_physics(); + t.run_bfb(); } } // namespace - diff --git a/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp index 0377b990d786..b108ce329d3e 100644 --- a/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp @@ -4,8 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" -#include "share/util/scream_setup_random_test.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -19,45 +18,49 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestBackToCellAverage { +struct UnitWrap::UnitTest::TestBackToCellAverage : public UnitWrap::UnitTest::Base { -static void run_phys() +void run_phys() { // TODO } -static void run_bfb() +void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); // Generate n test structs, each populated with random data (values within // [0,1]) by the default constructor. - BackToCellAverageData back_to_cell_average_data[Spack::n]; - for (Int i = 0; i < Spack::n; ++i) { - back_to_cell_average_data[i].randomize(engine); + BackToCellAverageData back_to_cell_average_data[max_pack_size]; + for (auto& item : back_to_cell_average_data) { + item.randomize(engine); } // Sync to device. - view_1d device_data("back_to_cell_average", Spack::n); + view_1d device_data("back_to_cell_average", max_pack_size); const auto host_data = Kokkos::create_mirror_view(device_data); - std::copy(&back_to_cell_average_data[0], &back_to_cell_average_data[0] + Spack::n, + std::copy(&back_to_cell_average_data[0], &back_to_cell_average_data[0] + max_pack_size, host_data.data()); Kokkos::deep_copy(device_data, host_data); - // Run the Fortran subroutine. - for (Int i = 0; i < Spack::n; ++i) { - back_to_cell_average(back_to_cell_average_data[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + back_to_cell_average_data[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host - Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + Kokkos::parallel_for(num_test_itrs, KOKKOS_LAMBDA(const Int& i) { + const Int offset = i * Spack::n; + // Init pack inputs Spack cld_frac_l, cld_frac_r, cld_frac_i, qc2qr_accret_tend, qr2qv_evap_tend, qc2qr_autoconv_tend, nc_accret_tend, nc_selfcollect_tend, nc2nr_autoconv_tend, nr_selfcollect_tend, nr_evap_tend, ncautr, qi2qv_sublim_tend, nr_ice_shed_tend, qc2qi_hetero_freeze_tend, qr2qi_collect_tend, qc2qr_ice_shed_tend, qi2qr_melt_tend, qc2qi_collect_tend, qr2qi_immers_freeze_tend, ni2nr_melt_tend, nc_collect_tend, ncshdc, nc2ni_immers_freeze_tend, nr_collect_tend, ni_selfcollect_tend, qv2qi_vapdep_tend, nr2ni_immers_freeze_tend, ni_sublim_tend, qv2qi_nucleat_tend, ni_nucleat_tend, qc2qi_berg_tend; - for (Int s = 0; s < Spack::n; ++s) { + for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { cld_frac_l[s] = device_data[s].cld_frac_l; cld_frac_r[s] = device_data[s].cld_frac_r; cld_frac_i[s] = device_data[s].cld_frac_i; @@ -99,7 +102,7 @@ static void run_bfb() nr_collect_tend, ni_selfcollect_tend, qv2qi_vapdep_tend, nr2ni_immers_freeze_tend, ni_sublim_tend, qv2qi_nucleat_tend, ni_nucleat_tend, qc2qi_berg_tend); // Copy results back into views - for (Int s = 0; s < Spack::n; ++s) { + for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { device_data(s).qc2qr_accret_tend = qc2qr_accret_tend[s]; device_data(s).qr2qv_evap_tend = qr2qv_evap_tend[s]; device_data(s).qc2qr_autoconv_tend = qc2qr_autoconv_tend[s]; @@ -136,8 +139,8 @@ static void run_bfb() Kokkos::deep_copy(host_data, device_data); // Validate results. - if (SCREAM_BFB_TESTING) { - for (Int s = 0; s < Spack::n; ++s) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(back_to_cell_average_data[s].qc2qr_accret_tend == host_data[s].qc2qr_accret_tend); REQUIRE(back_to_cell_average_data[s].qr2qv_evap_tend == host_data[s].qr2qv_evap_tend); REQUIRE(back_to_cell_average_data[s].qc2qr_autoconv_tend == host_data[s].qc2qr_autoconv_tend); @@ -169,6 +172,11 @@ static void run_bfb() REQUIRE(back_to_cell_average_data[s].qc2qi_berg_tend == host_data[s].qc2qi_berg_tend); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + host_data(s).write(Base::m_fid); + } + } } }; @@ -181,12 +189,11 @@ namespace { TEST_CASE("p3_back_to_cell_average", "[p3_functions]") { - using TRIF = scream::p3::unit_test::UnitWrap::UnitTest::TestBackToCellAverage; - - TRIF::run_phys(); - TRIF::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestBackToCellAverage; - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp index 97193fd06d31..cf302d78166c 100644 --- a/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp @@ -4,8 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" -#include "share/util/scream_setup_random_test.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -20,22 +19,19 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestCalcLiqRelaxationTimescale { +struct UnitWrap::UnitTest::TestCalcLiqRelaxationTimescale : public UnitWrap::UnitTest::Base { - static void run_phys() + void run_phys() { // TODO } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); // Read in tables - view_2d_table vn_table_vals, vm_table_vals, revap_table_vals; - view_1d_table mu_r_table_vals; - view_dnu_table dnu; - Functions::init_kokkos_tables(vn_table_vals, vm_table_vals, revap_table_vals, mu_r_table_vals, dnu); + auto revap_table_vals = Functions::p3_init().revap_table_vals; using KTH = KokkosTypes; @@ -54,9 +50,11 @@ struct UnitWrap::UnitTest::TestCalcLiqRelaxationTimescale { self[i].f2r = C::f2r; } - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - calc_liq_relaxation_timescale(self[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + self[i].read(Base::m_fid); + } } // Sync to device @@ -97,12 +95,17 @@ struct UnitWrap::UnitTest::TestCalcLiqRelaxationTimescale { Kokkos::deep_copy(self_host, self_device); - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(self[s].epsr == self_host(s).epsr); REQUIRE(self[s].epsc == self_host(s).epsc); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + self_host(s).write(Base::m_fid); + } + } } }; @@ -115,10 +118,11 @@ namespace { TEST_CASE("p3_calc_liq_relaxation_timescale", "[p3_functions]") { - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestCalcLiqRelaxationTimescale; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestCalcLiqRelaxationTimescale; - TD::run_phys(); - TD::run_bfb(); + T t; + t.run_phys(); + t.run_bfb(); } } diff --git a/components/eamxx/src/physics/p3/tests/p3_calc_rime_density_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_calc_rime_density_unit_tests.cpp index 5d70c7302ae1..9b7722895faf 100644 --- a/components/eamxx/src/physics/p3/tests/p3_calc_rime_density_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_calc_rime_density_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -18,14 +18,14 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestCalcRimeDensity { +struct UnitWrap::UnitTest::TestCalcRimeDensity : public UnitWrap::UnitTest::Base { -static void run_phys() +void run_phys() { // TODO } -static void run_bfb() +void run_bfb() { // This is the threshold for whether the qc cloud mixing ratio is large // enough to affect the rime density. @@ -87,9 +87,11 @@ static void run_bfb() host_data.data()); Kokkos::deep_copy(device_data, host_data); - // Run the Fortran subroutine. - for (Int i = 0; i < max_pack_size; ++i) { - calc_rime_density(calc_rime_density_data[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + calc_rime_density_data[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -126,12 +128,17 @@ static void run_bfb() Kokkos::deep_copy(host_data, device_data); // Validate results. - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(calc_rime_density_data[s].vtrmi1 == host_data[s].vtrmi1); REQUIRE(calc_rime_density_data[s].rho_qm_cloud == host_data[s].rho_qm_cloud); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + host_data(s).write(Base::m_fid); + } + } } }; @@ -144,12 +151,11 @@ namespace { TEST_CASE("p3_calc_rime_density", "[p3_functions]") { - using TRIF = scream::p3::unit_test::UnitWrap::UnitTest::TestCalcRimeDensity; - - TRIF::run_phys(); - TRIF::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestCalcRimeDensity; - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_check_values_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_check_values_unit_tests.cpp index f2f4a7f44047..90c8d28be932 100644 --- a/components/eamxx/src/physics/p3/tests/p3_check_values_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_check_values_unit_tests.cpp @@ -4,9 +4,8 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" -#include "p3_f90.hpp" -#include "share/util/scream_setup_random_test.hpp" +#include "p3_test_data.hpp" +#include "p3_data.hpp" #include "p3_unit_tests_common.hpp" @@ -20,13 +19,14 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestCheckValues { +struct UnitWrap::UnitTest::TestCheckValues : public UnitWrap::UnitTest::Base { -static void run_check_values_bfb() +void run_check_values_bfb() { + // This is not really a bfb test since no results are being checked. auto engine = setup_random_test(); - CheckValuesData cvd_fortran[] = { + CheckValuesData cvd_cxx[] = { // kts_, kte_, timestepcount_, source_ind_, force_abort_ CheckValuesData(1, 72, 2, 100, false), CheckValuesData(1, 72, 3, 100, false), @@ -34,33 +34,17 @@ static void run_check_values_bfb() CheckValuesData(1, 72, 5, 100, false), }; - static constexpr Int num_runs = sizeof(cvd_fortran) / sizeof(CheckValuesData); - - for (auto& d : cvd_fortran) { + for (auto& d : cvd_cxx) { d.randomize(engine, { {d.qv, {-4.056E-01, 1.153E+00}}, {d.temp, {1.000E+02, 5.000E+02}} }); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that - // inout data is in original state - CheckValuesData cvd_cxx[num_runs] = { - CheckValuesData(cvd_fortran[0]), - CheckValuesData(cvd_fortran[1]), - CheckValuesData(cvd_fortran[2]), - CheckValuesData(cvd_fortran[3]), - }; - - // Get data from fortran - for (auto& d : cvd_fortran) { - check_values(d); - } - // Get data from cxx for (auto& d : cvd_cxx) { - check_values_f(d.qv, d.temp, d.kts, d.kte, d.timestepcount, d.force_abort, d.source_ind, d.col_loc); + check_values_host(d.qv, d.temp, d.kts, d.kte, d.timestepcount, d.force_abort, d.source_ind, d.col_loc); } } -static void run_check_values_phys() +void run_check_values_phys() { // TODO } @@ -75,14 +59,11 @@ namespace { TEST_CASE("p3_check_values", "[p3_functions]") { - using TRS = scream::p3::unit_test::UnitWrap::UnitTest::TestCheckValues; - - scream::p3::p3_init(); // need fortran table data - - TRS::run_check_values_phys(); - TRS::run_check_values_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestCheckValues; - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_check_values_phys(); + t.run_check_values_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp index a9267449541c..bbaa95e87a03 100644 --- a/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -18,14 +18,14 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestCldliqImmersionFreezing { +struct UnitWrap::UnitTest::TestCldliqImmersionFreezing : public UnitWrap::UnitTest::Base { -static void run_phys() +void run_phys() { // TODO } -static void run_bfb() +void run_bfb() { // This is the threshold for whether the qc and qr cloud mixing ratios are // large enough to affect the warm-phase process rates qc2qr_accret_tend and nc_accret_tend. @@ -70,9 +70,11 @@ static void run_bfb() host_data.data()); Kokkos::deep_copy(device_data, host_data); - // Run the Fortran subroutine. - for (Int i = 0; i < max_pack_size; ++i) { - cldliq_immersion_freezing(cldliq_imm_freezing_data[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + cldliq_imm_freezing_data[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -93,8 +95,10 @@ static void run_bfb() Spack qc2qi_hetero_freeze_tend{0.0}; Spack nc2ni_immers_freeze_tend{0.0}; - Functions::cldliq_immersion_freezing(T_atm, lamc, mu_c, cdist1, qc_incld, inv_qc_relvar, - qc2qi_hetero_freeze_tend, nc2ni_immers_freeze_tend, physics::P3_Constants()); + Functions::cldliq_immersion_freezing( + T_atm, lamc, mu_c, cdist1, qc_incld, inv_qc_relvar, + qc2qi_hetero_freeze_tend, nc2ni_immers_freeze_tend, + p3::Functions::P3Runtime()); // Copy results back into views for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { @@ -107,12 +111,17 @@ static void run_bfb() Kokkos::deep_copy(host_data, device_data); // Validate results. - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(cldliq_imm_freezing_data[s].qc2qi_hetero_freeze_tend == host_data[s].qc2qi_hetero_freeze_tend); REQUIRE(cldliq_imm_freezing_data[s].nc2ni_immers_freeze_tend == host_data[s].nc2ni_immers_freeze_tend); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + host_data(s).write(Base::m_fid); + } + } } }; @@ -125,12 +134,11 @@ namespace { TEST_CASE("p3_cldliq_immersion_freezing", "[p3_functions]") { - using TRIF = scream::p3::unit_test::UnitWrap::UnitTest::TestCldliqImmersionFreezing; - - TRIF::run_phys(); - TRIF::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestCldliqImmersionFreezing; - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp index 6b9ed9f63181..271f7e27bd61 100644 --- a/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -18,14 +18,14 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestCloudRainAccretion { +struct UnitWrap::UnitTest::TestCloudRainAccretion : public UnitWrap::UnitTest::Base { -static void run_phys() +void run_phys() { // TODO } -static void run_bfb() +void run_bfb() { // This is the threshold for whether the qc and qr cloud mixing ratios are // large enough to affect the warm-phase process rates qc2qr_accret_tend and nc_accret_tend. @@ -73,9 +73,11 @@ static void run_bfb() host_data.data()); Kokkos::deep_copy(device_data, host_data); - // Run the Fortran subroutine. - for (Int i = 0; i < max_pack_size; ++i) { - cloud_rain_accretion(cloud_rain_acc_data[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + cloud_rain_acc_data[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -96,8 +98,10 @@ static void run_bfb() Spack qc2qr_accret_tend{0.0}; Spack nc_accret_tend{0.0}; - Functions::cloud_rain_accretion(rho, inv_rho, qc_incld, nc_incld, qr_incld, - inv_qc_relvar, qc2qr_accret_tend, nc_accret_tend, physics::P3_Constants()); + Functions::cloud_rain_accretion( + rho, inv_rho, qc_incld, nc_incld, qr_incld, inv_qc_relvar, + qc2qr_accret_tend, nc_accret_tend, + p3::Functions::P3Runtime()); // Copy results back into views for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { @@ -110,12 +114,17 @@ static void run_bfb() Kokkos::deep_copy(host_data, device_data); // Validate results. - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(cloud_rain_acc_data[s].qc2qr_accret_tend == host_data[s].qc2qr_accret_tend); REQUIRE(cloud_rain_acc_data[s].nc_accret_tend == host_data[s].nc_accret_tend); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + host_data(s).write(Base::m_fid); + } + } } }; @@ -128,12 +137,11 @@ namespace { TEST_CASE("p3_cloud_rain_accretion", "[p3_functions]") { - using TCRA = scream::p3::unit_test::UnitWrap::UnitTest::TestCloudRainAccretion; - - TCRA::run_phys(); - TCRA::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestCloudRainAccretion; - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp index 19fe5b3279e7..b29bf11faa1c 100644 --- a/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp @@ -4,8 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" -#include "share/util/scream_setup_random_test.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -19,18 +18,18 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestCloudSed { +struct UnitWrap::UnitTest::TestCloudSed : public UnitWrap::UnitTest::Base { -static void run_phys() +void run_phys() { // TODO } -static void run_bfb() +void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - CloudSedData csds_fortran[] = { + CloudSedData csds_baseline[] = { // kts, kte, ktop, kbot, kdir, dt, inv_dt, do_predict_nc, precip_liq_surf, CloudSedData(1, 72, 27, 72, -1, 1.800E+03, 5.556E-04, false, 0.0), CloudSedData(1, 72, 72, 27, 1, 1.800E+03, 5.556E-04, false, 0.0), @@ -39,51 +38,58 @@ static void run_bfb() CloudSedData(1, 72, 27, 27, -1, 1.800E+03, 5.556E-04, true, 0.0), }; - static constexpr Int num_runs = sizeof(csds_fortran) / sizeof(CloudSedData); + static constexpr Int num_runs = sizeof(csds_baseline) / sizeof(CloudSedData); // Set up random input data - for (auto& d : csds_fortran) { + for (auto& d : csds_baseline) { d.randomize(engine, { {d.qc_incld, {C::QSMALL/2, C::QSMALL*2}} }); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state CloudSedData csds_cxx[num_runs] = { - CloudSedData(csds_fortran[0]), - CloudSedData(csds_fortran[1]), - CloudSedData(csds_fortran[2]), - CloudSedData(csds_fortran[3]), - CloudSedData(csds_fortran[4]), + CloudSedData(csds_baseline[0]), + CloudSedData(csds_baseline[1]), + CloudSedData(csds_baseline[2]), + CloudSedData(csds_baseline[3]), + CloudSedData(csds_baseline[4]), }; - // Get data from fortran - for (auto& d : csds_fortran) { - cloud_sedimentation(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : csds_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : csds_cxx) { - cloud_sedimentation_f(d.kts, d.kte, d.ktop, d.kbot, d.kdir, + cloud_sedimentation_host(d.kts, d.kte, d.ktop, d.kbot, d.kdir, d.qc_incld, d.rho, d.inv_rho, d.cld_frac_l, d.acn, d.inv_dz, d.dt, d.inv_dt, d.do_predict_nc, d.qc, d.nc, d.nc_incld, d.mu_c, d.lamc, &d.precip_liq_surf, d.qc_tend, d.nc_tend); } - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { // Due to pack issues, we must restrict checks to the active k space - Int start = std::min(csds_fortran[i].kbot, csds_fortran[i].ktop) - 1; // 0-based indx - Int end = std::max(csds_fortran[i].kbot, csds_fortran[i].ktop); // 0-based indx + Int start = std::min(csds_baseline[i].kbot, csds_baseline[i].ktop) - 1; // 0-based indx + Int end = std::max(csds_baseline[i].kbot, csds_baseline[i].ktop); // 0-based indx for (Int k = start; k < end; ++k) { - REQUIRE(csds_fortran[i].qc[k] == csds_cxx[i].qc[k]); - REQUIRE(csds_fortran[i].nc[k] == csds_cxx[i].nc[k]); - REQUIRE(csds_fortran[i].nc_incld[k] == csds_cxx[i].nc_incld[k]); - REQUIRE(csds_fortran[i].mu_c[k] == csds_cxx[i].mu_c[k]); - REQUIRE(csds_fortran[i].lamc[k] == csds_cxx[i].lamc[k]); - REQUIRE(csds_fortran[i].qc_tend[k] == csds_cxx[i].qc_tend[k]); - REQUIRE(csds_fortran[i].nc_tend[k] == csds_cxx[i].nc_tend[k]); + REQUIRE(csds_baseline[i].qc[k] == csds_cxx[i].qc[k]); + REQUIRE(csds_baseline[i].nc[k] == csds_cxx[i].nc[k]); + REQUIRE(csds_baseline[i].nc_incld[k] == csds_cxx[i].nc_incld[k]); + REQUIRE(csds_baseline[i].mu_c[k] == csds_cxx[i].mu_c[k]); + REQUIRE(csds_baseline[i].lamc[k] == csds_cxx[i].lamc[k]); + REQUIRE(csds_baseline[i].qc_tend[k] == csds_cxx[i].qc_tend[k]); + REQUIRE(csds_baseline[i].nc_tend[k] == csds_cxx[i].nc_tend[k]); } - REQUIRE(csds_fortran[i].precip_liq_surf == csds_cxx[i].precip_liq_surf); + REQUIRE(csds_baseline[i].precip_liq_surf == csds_cxx[i].precip_liq_surf); + } + } + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + csds_cxx[i].write(Base::m_fid); } } } @@ -98,12 +104,11 @@ namespace { TEST_CASE("p3_cloud_sed", "[p3_functions]") { - using TCS = scream::p3::unit_test::UnitWrap::UnitTest::TestCloudSed; - - TCS::run_phys(); - TCS::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestCloudSed; - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_droplet_self_coll_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_droplet_self_coll_unit_tests.cpp index b81505816eac..f183bba65da6 100644 --- a/components/eamxx/src/physics/p3/tests/p3_droplet_self_coll_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_droplet_self_coll_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -18,14 +18,14 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestDropletSelfCollection { +struct UnitWrap::UnitTest::TestDropletSelfCollection : public UnitWrap::UnitTest::Base { -static void run_phys() +void run_phys() { // TODO } -static void run_bfb() +void run_bfb() { // This is the threshold for whether the qc and qr cloud mixing ratios are // large enough to affect the warm-phase process rates qc2qr_accret_tend and nc_accret_tend. @@ -72,9 +72,11 @@ static void run_bfb() host_data.data()); Kokkos::deep_copy(device_data, host_data); - // Run the Fortran subroutine. - for (Int i = 0; i < max_pack_size; ++i) { - droplet_self_collection(droplet_self_coll_data[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + droplet_self_coll_data[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -107,11 +109,17 @@ static void run_bfb() Kokkos::deep_copy(host_data, device_data); // Validate results. - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(droplet_self_coll_data[s].nc_selfcollect_tend == host_data[s].nc_selfcollect_tend); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + host_data(s).write(Base::m_fid); + } + } + } }; @@ -124,12 +132,11 @@ namespace { TEST_CASE("p3_droplet_self_collection", "[p3_functions]") { - using TCRA = scream::p3::unit_test::UnitWrap::UnitTest::TestDropletSelfCollection; - - TCRA::run_phys(); - TCRA::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestDropletSelfCollection; - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp index 0e00cf5acddb..9eb783b0eb62 100644 --- a/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -22,14 +22,14 @@ namespace unit_test { */ template -struct UnitWrap::UnitTest::TestDsd2 { +struct UnitWrap::UnitTest::TestDsd2 : public UnitWrap::UnitTest::Base { - static void run_cloud_bfb() + void run_cloud_bfb() { // Read in tables view_2d_table vn_table_vals; view_2d_table vm_table_vals; view_2d_table revap_table_vals; view_1d_table mu_r_table_vals; view_dnu_table dnu; - Functions::init_kokkos_tables(vn_table_vals, vm_table_vals, revap_table_vals, mu_r_table_vals, dnu); + Functions::get_global_tables(vn_table_vals, vm_table_vals, revap_table_vals, mu_r_table_vals, dnu); // Load some lookup inputs, need at least one per pack value GetCloudDsd2Data gcdd[max_pack_size] = { @@ -60,9 +60,11 @@ struct UnitWrap::UnitTest::TestDsd2 { std::copy(&gcdd[0], &gcdd[0] + max_pack_size, gcdd_host.data()); Kokkos::deep_copy(gcdd_device, gcdd_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - get_cloud_dsd2(gcdd[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + gcdd[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -95,7 +97,7 @@ struct UnitWrap::UnitTest::TestDsd2 { Kokkos::deep_copy(gcdd_host, gcdd_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(gcdd[s].nc_out == gcdd_host(s).nc_out); REQUIRE(gcdd[s].mu_c == gcdd_host(s).mu_c); @@ -105,14 +107,19 @@ struct UnitWrap::UnitTest::TestDsd2 { REQUIRE(gcdd[s].cdist1 == gcdd_host(s).cdist1); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + gcdd_host(s).write(Base::m_fid); + } + } } - static void run_cloud_phys() + void run_cloud_phys() { // TODO } - static void run_rain_bfb() + void run_rain_bfb() { using KTH = KokkosTypes; @@ -144,9 +151,11 @@ struct UnitWrap::UnitTest::TestDsd2 { std::copy(&grdd[0], &grdd[0] + max_pack_size, grdd_host.data()); Kokkos::deep_copy(grdd_device, grdd_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - get_rain_dsd2(grdd[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + grdd[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -161,7 +170,8 @@ struct UnitWrap::UnitTest::TestDsd2 { } Spack mu_r(0.0), lamr(0.0), cdistr(0.0), logn0r(0.0); - Functions::get_rain_dsd2(qr, nr, mu_r, lamr, physics::P3_Constants()); + Functions::get_rain_dsd2(qr, nr, mu_r, lamr, + p3::Functions::P3Runtime()); Functions::get_cdistr_logn0r(qr, nr, mu_r, lamr, cdistr, logn0r); // Copy results back into views @@ -178,7 +188,7 @@ struct UnitWrap::UnitTest::TestDsd2 { Kokkos::deep_copy(grdd_host, grdd_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(grdd[s].nr_out == grdd_host(s).nr_out); REQUIRE(grdd[s].mu_r == grdd_host(s).mu_r); @@ -187,9 +197,14 @@ struct UnitWrap::UnitTest::TestDsd2 { REQUIRE(grdd[s].logn0r == grdd_host(s).logn0r); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + grdd_host(s).write(Base::m_fid); + } + } } - static void run_rain_phys() + void run_rain_phys() { // TODO } @@ -203,18 +218,20 @@ namespace { TEST_CASE("p3_cloud_dsd2", "[p3_functions]") { - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestDsd2; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestDsd2; - TD::run_cloud_phys(); - TD::run_cloud_bfb(); + T t; + t.run_cloud_phys(); + t.run_cloud_bfb(); } TEST_CASE("p3_rain_dsd2", "[p3_functions]") { - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestDsd2; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestDsd2; - TD::run_rain_phys(); - TD::run_rain_bfb(); + T t; + t.run_rain_phys(); + t.run_rain_bfb(); } } diff --git a/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp index f8398135a19e..a84b1e0448ad 100644 --- a/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp @@ -4,25 +4,18 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" -//#include -//#include -//#include -//#include -//#include // std::setprecision - namespace scream { namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestEvapSublPrecip -{ +struct UnitWrap::UnitTest::TestEvapSublPrecip : public UnitWrap::UnitTest::Base { - static void run_property(){ + void run_property() { //TEST WEIGHTING TIMESCALE //======================== @@ -133,13 +126,13 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip REQUIRE( qrtend[0] <= qr_incld[0]/dt); REQUIRE( nrtend[0] <= nr_incld[0]/dt); //keep end-of-step nr positive. Should always be true. - }; //end run_property + } //end run_property - static void run_bfb(){ + void run_bfb() { constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; - //fortran generated data is input to the following + //baseline generated data is input to the following //This subroutine has 20 args, only 18 are supplied here for invoking it as last 2 are intent-outs //note that dt is the same val for each row - this is needed since dt is a scalar and all rows are executed simultaneously on CPU in C++. //row1: above freezing, should trigger @@ -180,9 +173,11 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip std::copy(&espd[0], &espd[0] + max_pack_size, espd_host.data()); Kokkos::deep_copy(espd_device, espd_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - evaporate_rain(espd[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + espd[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -248,12 +243,17 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip Kokkos::deep_copy(espd_host, espd_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(espd[s].qr2qv_evap_tend == espd_host(s).qr2qv_evap_tend); REQUIRE(espd[s].nr_evap_tend == espd_host(s).nr_evap_tend); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + espd_host(s).write(Base::m_fid); + } + } } // end run_bfb @@ -267,14 +267,18 @@ namespace { TEST_CASE("p3_evaporate_rain_property", "p3_unit_tests") { - using TestStruct = scream::p3::unit_test::UnitWrap::UnitTest::TestEvapSublPrecip; - TestStruct::run_property(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestEvapSublPrecip; + + T t; + t.run_property(); } TEST_CASE("p3_evaporate_rain_test", "p3_unit_tests") { - using TestStruct = scream::p3::unit_test::UnitWrap::UnitTest::TestEvapSublPrecip; - TestStruct::run_bfb(); -} + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestEvapSublPrecip; + + T t; + t.run_bfb(); + } }// end anonymous namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_find_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_find_unit_tests.cpp index 82ca006d76da..6e8e09226b7f 100644 --- a/components/eamxx/src/physics/p3/tests/p3_find_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_find_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -22,13 +22,13 @@ namespace unit_test { // template -struct UnitWrap::UnitTest::TestFind { +struct UnitWrap::UnitTest::TestFind : public UnitWrap::UnitTest::Base { -static void run() +void run() { const int max_threads = #ifdef KOKKOS_ENABLE_OPENMP - Kokkos::OpenMP::concurrency() + Kokkos::OpenMP().concurrency() #else 1 #endif @@ -112,7 +112,10 @@ namespace { TEST_CASE("p3_find", "[p3_functions]") { - scream::p3::unit_test::UnitWrap::UnitTest::TestFind::run(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestFind; + + T t; + t.run(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_get_latent_heat_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_get_latent_heat_unit_tests.cpp deleted file mode 100644 index 5d7b4de4ea7e..000000000000 --- a/components/eamxx/src/physics/p3/tests/p3_get_latent_heat_unit_tests.cpp +++ /dev/null @@ -1,78 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "p3_functions.hpp" -#include "p3_functions_f90.hpp" -#include "p3_f90.hpp" - -#include "p3_unit_tests_common.hpp" - -#include -#include -#include -#include -#include // std::setprecision - -namespace scream { -namespace p3 { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestLatentHeat { - - static void run_latent_heat_bfb() - { - constexpr Scalar latvap = C::LatVap; - constexpr Scalar latice = C::LatIce; - - LatentHeatData latent_fortran[] = { - // its, ite, kts, kte - LatentHeatData(1, 7, 1, 10), - }; - - static constexpr Int num_runs = sizeof(latent_fortran) / sizeof(LatentHeatData); - - LatentHeatData latent_cxx[num_runs] = { - LatentHeatData(latent_fortran[0]), - LatentHeatData(latent_fortran[1]), - LatentHeatData(latent_fortran[2]), - LatentHeatData(latent_fortran[3]), - }; - - for (Int i = 0; i < num_runs; ++i) { - LatentHeatData& h = latent_fortran[i]; - get_latent_heat(h); - - if (SCREAM_BFB_TESTING) { - for (Int j = 0; j < h.total(h.v); ++j) { - REQUIRE(h.v[j] == latvap); - REQUIRE(h.s[j] == (latvap+latice)); - REQUIRE(h.f[j] == latice); - } - } - } - } - - static void run_latent_heat_phys() - { - // TODO - } -}; - -} -} -} - -namespace { - -TEST_CASE("p3_latent_heat", "[p3_functions]") -{ - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestLatentHeat; - - TD::run_latent_heat_phys(); - TD::run_latent_heat_bfb(); -} - -} diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp index caf6638d5cd5..cca204b9b3a3 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -19,9 +19,9 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestIceCldliqWetGrowth { +struct UnitWrap::UnitTest::TestIceCldliqWetGrowth : public UnitWrap::UnitTest::Base { - static void run_ice_cldliq_wet_growth_bfb() + void run_ice_cldliq_wet_growth_bfb() { using KTH = KokkosTypes; @@ -57,9 +57,11 @@ struct UnitWrap::UnitTest::TestIceCldliqWetGrowth { std::copy(&self[0], &self[0] + max_pack_size, self_host.data()); Kokkos::deep_copy(self_device, self_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - ice_cldliq_wet_growth(self[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + self[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -114,7 +116,7 @@ struct UnitWrap::UnitTest::TestIceCldliqWetGrowth { Kokkos::deep_copy(self_host, self_device); - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(static_cast(self[s].log_wetgrowth) == static_cast(self_host(s).log_wetgrowth)); @@ -125,9 +127,14 @@ struct UnitWrap::UnitTest::TestIceCldliqWetGrowth { REQUIRE(self[s].qc2qr_ice_shed_tend == self_host(s).qc2qr_ice_shed_tend); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + self_host(s).write(Base::m_fid); + } + } } - static void run_ice_cldliq_wet_growth_phys() + void run_ice_cldliq_wet_growth_phys() { // TODO } @@ -141,10 +148,11 @@ namespace { TEST_CASE("p3_ice_cldliq_wet_growth", "[p3_functions]") { - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestIceCldliqWetGrowth; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIceCldliqWetGrowth; - TD::run_ice_cldliq_wet_growth_phys(); - TD::run_ice_cldliq_wet_growth_bfb(); + T t; + t.run_ice_cldliq_wet_growth_phys(); + t.run_ice_cldliq_wet_growth_bfb(); } } diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp index bfad594acfb1..5d36ee79d546 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -22,16 +22,16 @@ namespace unit_test { * Unit-tests for p3 ice collection functions. */ template -struct UnitWrap::UnitTest::TestIceCollection { +struct UnitWrap::UnitTest::TestIceCollection : public UnitWrap::UnitTest::Base { - static void run_ice_cldliq_bfb() + void run_ice_cldliq_bfb() { // Read in tables view_2d_table vn_table_vals; view_2d_table vm_table_vals; view_2d_table revap_table_vals; view_1d_table mu_r_table_vals; view_dnu_table dnu; - Functions::init_kokkos_tables(vn_table_vals, vm_table_vals, revap_table_vals, mu_r_table_vals, dnu); + Functions::get_global_tables(vn_table_vals, vm_table_vals, revap_table_vals, mu_r_table_vals, dnu); // Load some lookup inputs, need at least one per pack value IceCldliqCollectionData cldliq[max_pack_size] = { @@ -63,9 +63,11 @@ struct UnitWrap::UnitTest::TestIceCollection { std::copy(&cldliq[0], &cldliq[0] + max_pack_size, cldliq_host.data()); Kokkos::deep_copy(cldliq_device, cldliq_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - ice_cldliq_collection(cldliq[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + cldliq[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -90,9 +92,11 @@ struct UnitWrap::UnitTest::TestIceCollection { Spack qc2qr_ice_shed_tend{0.0}; Spack ncshdc{0.0}; - Functions::ice_cldliq_collection(rho, temp, rhofaci, table_val_qc2qi_collect, qi_incld, - qc_incld, ni_incld, nc_incld, - qc2qi_collect_tend, nc_collect_tend, qc2qr_ice_shed_tend, ncshdc, physics::P3_Constants()); + Functions::ice_cldliq_collection( + rho, temp, rhofaci, table_val_qc2qi_collect, qi_incld, qc_incld, + ni_incld, nc_incld, qc2qi_collect_tend, nc_collect_tend, + qc2qr_ice_shed_tend, ncshdc, + p3::Functions::P3Runtime()); // Copy results back into views for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { @@ -107,7 +111,7 @@ struct UnitWrap::UnitTest::TestIceCollection { Kokkos::deep_copy(cldliq_host, cldliq_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(cldliq[s].qc2qi_collect_tend == cldliq_host(s).qc2qi_collect_tend); REQUIRE(cldliq[s].nc_collect_tend == cldliq_host(s).nc_collect_tend); @@ -115,14 +119,19 @@ struct UnitWrap::UnitTest::TestIceCollection { REQUIRE(cldliq[s].ncshdc == cldliq_host(s).ncshdc); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + cldliq_host(s).write(Base::m_fid); + } + } } - static void run_ice_cldliq_phys() + void run_ice_cldliq_phys() { // TODO } - static void run_ice_rain_bfb() + void run_ice_rain_bfb() { using KTH = KokkosTypes; @@ -155,9 +164,11 @@ struct UnitWrap::UnitTest::TestIceCollection { std::copy(&rain[0], &rain[0] + max_pack_size, rain_host.data()); Kokkos::deep_copy(rain_device, rain_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - ice_rain_collection(rain[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + rain[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -179,9 +190,11 @@ struct UnitWrap::UnitTest::TestIceCollection { } Spack qr2qi_collect_tend(0.0), nr_collect_tend(0.0); - Functions::ice_rain_collection(rho, temp, rhofaci, logn0r, table_val_nr_collect, table_val_qr2qi_collect, - qi_incld, ni_incld, qr_incld, - qr2qi_collect_tend, nr_collect_tend, physics::P3_Constants()); + Functions::ice_rain_collection( + rho, temp, rhofaci, logn0r, table_val_nr_collect, + table_val_qr2qi_collect, qi_incld, ni_incld, qr_incld, + qr2qi_collect_tend, nr_collect_tend, + p3::Functions::P3Runtime()); // Copy results back into views for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { @@ -194,20 +207,25 @@ struct UnitWrap::UnitTest::TestIceCollection { Kokkos::deep_copy(rain_host, rain_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(rain[s].qr2qi_collect_tend == rain_host(s).qr2qi_collect_tend); REQUIRE(rain[s].nr_collect_tend == rain_host(s).nr_collect_tend); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + rain_host(s).write(Base::m_fid); + } + } } - static void run_ice_rain_phys() + void run_ice_rain_phys() { // TODO } - static void run_ice_self_bfb() + void run_ice_self_bfb() { using KTH = KokkosTypes; @@ -240,9 +258,11 @@ struct UnitWrap::UnitTest::TestIceCollection { std::copy(&self[0], &self[0] + max_pack_size, self_host.data()); Kokkos::deep_copy(self_device, self_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - ice_self_collection(self[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + self[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -272,15 +292,19 @@ struct UnitWrap::UnitTest::TestIceCollection { Kokkos::deep_copy(self_host, self_device); - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(self[s].ni_selfcollect_tend == self_host(s).ni_selfcollect_tend); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + self_host(s).write(Base::m_fid); + } + } } - - static void run_ice_self_phys() + void run_ice_self_phys() { // TODO } @@ -294,24 +318,29 @@ namespace { TEST_CASE("p3_ice_cldliq", "[p3_functions]") { - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestIceCollection; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIceCollection; - TD::run_ice_cldliq_phys(); - TD::run_ice_cldliq_bfb(); + T t; + t.run_ice_cldliq_phys(); + t.run_ice_cldliq_bfb(); } TEST_CASE("p3_ice_rain", "[p3_functions]") { - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestIceCollection; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIceCollection; - TD::run_ice_rain_phys(); - TD::run_ice_rain_bfb(); + T t; + t.run_ice_rain_phys(); + t.run_ice_rain_bfb(); } TEST_CASE("p3_ice_self", "[p3_functions]") { - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestIceCollection; - TD::run_ice_self_phys(); - TD::run_ice_self_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIceCollection; + + T t; + t.run_ice_self_phys(); + t.run_ice_self_bfb(); } + } diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_deposition_sublimation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_deposition_sublimation_tests.cpp index bc1921441bb3..9b261409ad09 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_deposition_sublimation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_deposition_sublimation_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "p3_unit_tests_common.hpp" @@ -13,9 +13,9 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestIceDepositionSublimation { +struct UnitWrap::UnitTest::TestIceDepositionSublimation : public UnitWrap::UnitTest::Base { - static void run_property(){ + void run_property() { //Note that a lot of property tests are included in run_bfb for simplicity //Choose default values (just grabbed a row from the array in run_bfb) @@ -30,13 +30,13 @@ struct UnitWrap::UnitTest::TestIceDepositionSublimation { //init output vars Spack qv2qi_vapdep_tend, qi2qv_sublim_tend, ni_sublim_tend, qc2qi_berg_tend; - + //CHECK THAT UNREASONABLY LARGE VAPOR DEPOSITION DOESN'T LEAVE QV SUBSATURATED WRT QI Spack epsi_tmp=1e6; //make 1/(sat removal timescale) huge so vapdep rate removes all supersat in 1 dt. Functions::ice_deposition_sublimation(qi_incld, ni_incld, T_atm, qv_sat_l, qv_sat_i, epsi_tmp, abi, qv, inv_dt, qv2qi_vapdep_tend, qi2qv_sublim_tend, ni_sublim_tend, qc2qi_berg_tend); REQUIRE( (qv2qi_vapdep_tend[0]==0 || std::abs( qv2qi_vapdep_tend[0] - (qv[0] - qv_sat_i[0])*inv_dt) <1e-8) ); - + //CHECK THAT HUGE SUBLIMATION DOESN'T LEAVE QV SUPERSATURATED WRT QI Spack qv_sat_i_tmp=1e-2; Functions::ice_deposition_sublimation(qi_incld, ni_incld, T_atm, qv_sat_l, qv_sat_i_tmp, @@ -46,10 +46,10 @@ struct UnitWrap::UnitTest::TestIceDepositionSublimation { //CHECK BEHAVIOR AS DT->0? } - - static void run_bfb() + + void run_bfb() { - IceDepositionSublimationData f90_data[max_pack_size] = { + IceDepositionSublimationData baseline_data[max_pack_size] = { {1.0000E-04,4.5010E+05,2.8750E+02,1.1279E-02,1.1279E-02,0.0000E+00,3.3648E+00,5.0000E-03,1.666667e-02}, {5.1000E-03,4.5370E+05,2.8542E+02,9.9759E-03,9.9759E-03,0.0000E+00,3.1223E+00,5.0000E-03,1.666667e-02}, {5.1000E-03,4.5742E+05,2.8334E+02,8.8076E-03,8.8076E-03,0.0000E+00,2.9014E+00,5.0000E-03,1.666667e-02}, @@ -67,25 +67,25 @@ struct UnitWrap::UnitTest::TestIceDepositionSublimation { {5.1000E-03,5.1317E+05,2.5834E+02,1.6757E-03,1.4491E-03,6.0620E-02,1.3763E+00,5.0000E-03,1.666667e-02}, {5.0000E-08,5.4479E+05,2.4793E+02,7.5430E-04,5.8895E-04,4.6769E-04,1.1661E+00,1.5278E-04,1.666667e-02}, }; - - static constexpr Int num_runs = sizeof(f90_data) / sizeof(IceDepositionSublimationData); // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - //for (auto& d : f90_data) { + // Alternatively, you can use the baseline_data construtors/initializer lists to hardcode data + //for (auto& d : baseline_data) { // d.randomize(); //} - // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that + // Create copies of data for use by cxx and sync it to device. Needs to happen before reads so that // inout data is in original state view_1d cxx_device("cxx_device", max_pack_size); const auto cxx_host = Kokkos::create_mirror_view(cxx_device); - std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); + std::copy(&baseline_data[0], &baseline_data[0] + max_pack_size, cxx_host.data()); Kokkos::deep_copy(cxx_device, cxx_host); - // Get data from fortran - for (auto& d : f90_data) { - ice_deposition_sublimation(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + baseline_data[i].read(Base::m_fid); + } } // Get data from cxx. Run ice_deposition_sublimation from a kernel and copy results back to host @@ -110,7 +110,6 @@ struct UnitWrap::UnitTest::TestIceDepositionSublimation { // Init outputs Spack ni_sublim_tend(0), qi2qv_sublim_tend(0), qc2qi_berg_tend(0), qv2qi_vapdep_tend(0); - Functions::ice_deposition_sublimation(qi_incld, ni_incld, T_atm, qv_sat_l, qv_sat_i, epsi, abi, qv, inv_dt, qv2qi_vapdep_tend, qi2qv_sublim_tend, ni_sublim_tend, qc2qi_berg_tend); // Copy spacks back into cxx_device view @@ -120,35 +119,40 @@ struct UnitWrap::UnitTest::TestIceDepositionSublimation { cxx_device(vs).qc2qi_berg_tend = qc2qi_berg_tend[s]; cxx_device(vs).qv2qi_vapdep_tend = qv2qi_vapdep_tend[s]; } - }); Kokkos::deep_copy(cxx_host, cxx_device); - for (Int i = 0; i < num_runs; ++i) { - // Verify BFB results - IceDepositionSublimationData& d_f90 = f90_data[i]; - IceDepositionSublimationData& d_cxx = cxx_host[i]; - REQUIRE(d_f90.qv2qi_vapdep_tend == d_cxx.qv2qi_vapdep_tend); - REQUIRE(d_f90.qi2qv_sublim_tend == d_cxx.qi2qv_sublim_tend); - REQUIRE(d_f90.ni_sublim_tend == d_cxx.ni_sublim_tend); - REQUIRE(d_f90.qc2qi_berg_tend == d_cxx.qc2qi_berg_tend); - - //MAKE SURE OUTPUT IS WITHIN EXPECTED BOUNDS: - REQUIRE(d_cxx.qv2qi_vapdep_tend >=0); - REQUIRE(d_cxx.qi2qv_sublim_tend >=0); - REQUIRE(d_cxx.ni_sublim_tend >=0); - REQUIRE(d_cxx.qc2qi_berg_tend >=0); - - //vapdep should only occur when qv>qv_sat_i - REQUIRE( (d_cxx.qv2qi_vapdep_tend==0 || d_cxx.qv + d_cxx.qv2qi_vapdep_tend*d_cxx.inv_dt >= d_cxx.qv_sat_i) ); - //sublim should only occur when qvfrz, berg and vapdep should be 0: - REQUIRE( (d_cxx.T_atmm_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + // Verify BFB results + IceDepositionSublimationData& d_f90 = baseline_data[i]; + IceDepositionSublimationData& d_cxx = cxx_host[i]; + REQUIRE(d_f90.qv2qi_vapdep_tend == d_cxx.qv2qi_vapdep_tend); + REQUIRE(d_f90.qi2qv_sublim_tend == d_cxx.qi2qv_sublim_tend); + REQUIRE(d_f90.ni_sublim_tend == d_cxx.ni_sublim_tend); + REQUIRE(d_f90.qc2qi_berg_tend == d_cxx.qc2qi_berg_tend); + + //MAKE SURE OUTPUT IS WITHIN EXPECTED BOUNDS: + REQUIRE(d_cxx.qv2qi_vapdep_tend >=0); + REQUIRE(d_cxx.qi2qv_sublim_tend >=0); + REQUIRE(d_cxx.ni_sublim_tend >=0); + REQUIRE(d_cxx.qc2qi_berg_tend >=0); + + //vapdep should only occur when qv>qv_sat_i + REQUIRE( (d_cxx.qv2qi_vapdep_tend==0 || d_cxx.qv + d_cxx.qv2qi_vapdep_tend*d_cxx.inv_dt >= d_cxx.qv_sat_i) ); + //sublim should only occur when qvfrz, berg and vapdep should be 0: + REQUIRE( (d_cxx.T_atmm_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + cxx_host(s).write(Base::m_fid); + } } } // run_bfb @@ -162,16 +166,18 @@ namespace { TEST_CASE("ice_deposition_sublimation_property", "[p3]") { - using TestStruct = scream::p3::unit_test::UnitWrap::UnitTest::TestIceDepositionSublimation; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIceDepositionSublimation; - TestStruct::run_property(); + T t; + t.run_property(); } - + TEST_CASE("ice_deposition_sublimation_bfb", "[p3]") { - using TestStruct = scream::p3::unit_test::UnitWrap::UnitTest::TestIceDepositionSublimation; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIceDepositionSublimation; - TestStruct::run_bfb(); + T t; + t.run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp index 28c6582f8a14..4e24cc8be0e0 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -19,10 +19,10 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestP3IceMelting +struct UnitWrap::UnitTest::TestP3IceMelting : public UnitWrap::UnitTest::Base { -static void ice_melting_bfb(){ +void ice_melting_bfb() { constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; @@ -57,9 +57,11 @@ static void ice_melting_bfb(){ std::copy(&IceMelt[0], &IceMelt[0] + max_pack_size, IceMelt_host.data()); Kokkos::deep_copy(IceMelt_device, IceMelt_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - ice_melting(IceMelt[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + IceMelt[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -99,13 +101,19 @@ static void ice_melting_bfb(){ Kokkos::deep_copy(IceMelt_host, IceMelt_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(IceMelt[s].qi2qr_melt_tend == IceMelt_host(s).qi2qr_melt_tend); REQUIRE(IceMelt[s].ni2nr_melt_tend == IceMelt_host(s).ni2nr_melt_tend); } } -}; // TestP3IceMelting + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + IceMelt_host(s).write(Base::m_fid); + } + } + +} }; // UnitWrap @@ -113,11 +121,13 @@ static void ice_melting_bfb(){ } // namespace p3 } // namespace scream -namespace{ +namespace { -TEST_CASE("p3_ice_melting_test", "[p3_ice_melting_test]"){ - scream::p3::unit_test::UnitWrap::UnitTest::TestP3IceMelting::ice_melting_bfb(); +TEST_CASE("p3_ice_melting_test", "[p3_ice_melting_test]") { + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestP3IceMelting; + + T t; + t.ice_melting_bfb(); } } // namespace - diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp index cb26a7ee6c06..7eeb4fa875ec 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -19,9 +19,9 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestIceNucleation { +struct UnitWrap::UnitTest::TestIceNucleation : public UnitWrap::UnitTest::Base { - static void run_ice_nucleation_bfb() + void run_ice_nucleation_bfb() { using KTH = KokkosTypes; @@ -54,10 +54,13 @@ struct UnitWrap::UnitTest::TestIceNucleation { {2.702E+02, 1.069E+00, 0.323E+03, 2.221E+01, 9.952E-01, inv_dt, do_predict_nc, do_prescribed_CCN } }; - // Run the fortran code - for (Int i = 0; i < max_pack_size; ++i) { - ice_nucleation(self[i]); - } + std::string root_name = "ice_nucleation"; + std::string file_name = root_name + (do_predict_nc ? "1" : "0") + (do_prescribed_CCN ? "1" : "0"); + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + self[i].read(Base::m_fid); + } + } // Sync to device KTH::view_1d self_host("self_host", max_pack_size); @@ -81,8 +84,10 @@ struct UnitWrap::UnitTest::TestIceNucleation { // outputs Spack qv2qi_nucleat_tend{0.0}; Spack ni_nucleat_tend{0.0}; - Functions::ice_nucleation(temp, inv_rho, ni, ni_activated, qv_supersat_i, self_device(0).inv_dt, do_predict_nc, - do_prescribed_CCN, qv2qi_nucleat_tend, ni_nucleat_tend, physics::P3_Constants()); + Functions::ice_nucleation( + temp, inv_rho, ni, ni_activated, qv_supersat_i, self_device(0).inv_dt, + do_predict_nc, do_prescribed_CCN, qv2qi_nucleat_tend, ni_nucleat_tend, + p3::Functions::P3Runtime()); for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { self_device(vs).qv2qi_nucleat_tend = qv2qi_nucleat_tend[s]; @@ -92,17 +97,22 @@ struct UnitWrap::UnitTest::TestIceNucleation { Kokkos::deep_copy(self_host, self_device); - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(self[s].qv2qi_nucleat_tend == self_host(s).qv2qi_nucleat_tend); REQUIRE(self[s].ni_nucleat_tend == self_host(s).ni_nucleat_tend); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + self_host(s).write(Base::m_fid); + } + } } //end for do_predict_nc } //end for do_prescribed_CCN } - static void run_ice_nucleation_phys() + void run_ice_nucleation_phys() { // TODO } @@ -116,10 +126,11 @@ namespace { TEST_CASE("p3_ice_nucleation", "[p3_functions]") { - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestIceNucleation; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIceNucleation; - TD::run_ice_nucleation_phys(); - TD::run_ice_nucleation_bfb(); + T t; + t.run_ice_nucleation_phys(); + t.run_ice_nucleation_bfb(); } } diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_relaxation_timescale_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_relaxation_timescale_unit_tests.cpp index a75b55fdbece..59768bbf82c6 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_relaxation_timescale_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_relaxation_timescale_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -19,9 +19,9 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestIceRelaxationTimescale { +struct UnitWrap::UnitTest::TestIceRelaxationTimescale : public UnitWrap::UnitTest::Base { - static void run_ice_relaxation_timescale_bfb() + void run_ice_relaxation_timescale_bfb() { using KTH = KokkosTypes; @@ -49,10 +49,12 @@ struct UnitWrap::UnitTest::TestIceRelaxationTimescale { {1.352E+01, 3.210E+03, 1.069E+00, 0.123E+00, 3.456E+00, 1.221E-02, 9.952E-07, 6.596E-05, 4.532E-01, 1.734E+04} }; - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - ice_relaxation_timescale(self[i]); - } + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + self[i].read(Base::m_fid); + } + } // Sync to device KTH::view_1d self_host("self_host", max_pack_size); @@ -93,15 +95,20 @@ struct UnitWrap::UnitTest::TestIceRelaxationTimescale { Kokkos::deep_copy(self_host, self_device); - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(self[s].epsi == self_host(s).epsi); REQUIRE(self[s].epsi_tot == self_host(s).epsi_tot); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + self_host(s).write(Base::m_fid); + } + } } - static void run_ice_relaxation_timescale_phys() + void run_ice_relaxation_timescale_phys() { // TODO } @@ -115,10 +122,11 @@ namespace { TEST_CASE("p3_ice_relaxation_timescale", "[p3_functions]") { - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestIceRelaxationTimescale; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIceRelaxationTimescale; - TD::run_ice_relaxation_timescale_phys(); - TD::run_ice_relaxation_timescale_bfb(); + T t; + t.run_ice_relaxation_timescale_phys(); + t.run_ice_relaxation_timescale_bfb(); } } diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp index 93c06a9054d7..faa6da1596ef 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp @@ -4,8 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" -#include "share/util/scream_setup_random_test.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -19,36 +18,36 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestIceSed { +struct UnitWrap::UnitTest::TestIceSed : public UnitWrap::UnitTest::Base { -static void run_phys_calc_bulk_rhime() +void run_phys_calc_bulk_rhime() { // TODO } -static void run_phys_ice_sed() +void run_phys_ice_sed() { // TODO } -static void run_phys_homogeneous_freezing() +void run_phys_homogeneous_freezing() { // TODO } -static void run_phys() +void run_phys() { run_phys_calc_bulk_rhime(); run_phys_ice_sed(); run_phys_homogeneous_freezing(); } -static void run_bfb_calc_bulk_rhime() +void run_bfb_calc_bulk_rhime() { constexpr Scalar qsmall = C::QSMALL; // Load some lookup inputs, need at least one per pack value - CalcBulkRhoRimeData cbrr_fortran[max_pack_size] = { + CalcBulkRhoRimeData cbrr_baseline[max_pack_size] = { // qi_tot, qi_rim, bi_rim {9.999978E-08, 9.999978E-03, 1.111108E-10}, {0.000000E+00, 8.571428E-05, 1.000000E-02}, @@ -71,17 +70,17 @@ static void run_bfb_calc_bulk_rhime() {5.164017E-10, 0.000000E+00, 0.000000E+00}, }; - // Sync to device, needs to happen before fortran calls so that + // Sync to device, needs to happen before reads so that // inout data is in original state view_1d cbrr_device("cbrr", max_pack_size); const auto cbrr_host = Kokkos::create_mirror_view(cbrr_device); - std::copy(&cbrr_fortran[0], &cbrr_fortran[0] + max_pack_size, cbrr_host.data()); + std::copy(&cbrr_baseline[0], &cbrr_baseline[0] + max_pack_size, cbrr_host.data()); Kokkos::deep_copy(cbrr_device, cbrr_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - if (cbrr_fortran[i].qi_tot > qsmall) { - calc_bulk_rho_rime(cbrr_fortran[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + cbrr_baseline[i].read(Base::m_fid); } } @@ -98,7 +97,9 @@ static void run_bfb_calc_bulk_rhime() } Smask gt_small(qi_tot > qsmall); - Spack rho_rime = Functions::calc_bulk_rho_rime(qi_tot, qi_rim, bi_rim, physics::P3_Constants(), gt_small); + Spack rho_rime = Functions::calc_bulk_rho_rime( + qi_tot, qi_rim, bi_rim, p3::Functions::P3Runtime(), + gt_small); // Copy results back into views for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { @@ -112,20 +113,25 @@ static void run_bfb_calc_bulk_rhime() Kokkos::deep_copy(cbrr_host, cbrr_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { - REQUIRE(cbrr_fortran[s].qi_rim == cbrr_host(s).qi_rim); - REQUIRE(cbrr_fortran[s].bi_rim == cbrr_host(s).bi_rim); - REQUIRE(cbrr_fortran[s].rho_rime == cbrr_host(s).rho_rime); + REQUIRE(cbrr_baseline[s].qi_rim == cbrr_host(s).qi_rim); + REQUIRE(cbrr_baseline[s].bi_rim == cbrr_host(s).bi_rim); + REQUIRE(cbrr_baseline[s].rho_rime == cbrr_host(s).rho_rime); + } + } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + cbrr_host(s).write(Base::m_fid); } } } -static void run_bfb_ice_sed() +void run_bfb_ice_sed() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - IceSedData isds_fortran[] = { + IceSedData isds_baseline[] = { // kts, kte, ktop, kbot, kdir, dt, inv_dt, precip_ice_surf IceSedData(1, 72, 27, 72, -1, 1.800E+03, 5.556E-04, 0.0), IceSedData(1, 72, 72, 27, 1, 1.800E+03, 5.556E-04, 1.0), @@ -133,65 +139,72 @@ static void run_bfb_ice_sed() IceSedData(1, 72, 27, 27, 1, 1.800E+03, 5.556E-04, 2.0), }; - static constexpr Int num_runs = sizeof(isds_fortran) / sizeof(IceSedData); + static constexpr Int num_runs = sizeof(isds_baseline) / sizeof(IceSedData); // Set up random input data - for (auto& d : isds_fortran) { + for (auto& d : isds_baseline) { d.randomize(engine, { {d.qi_incld, {C::QSMALL/2, C::QSMALL*2}} }); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state IceSedData isds_cxx[num_runs] = { - IceSedData(isds_fortran[0]), - IceSedData(isds_fortran[1]), - IceSedData(isds_fortran[2]), - IceSedData(isds_fortran[3]), + IceSedData(isds_baseline[0]), + IceSedData(isds_baseline[1]), + IceSedData(isds_baseline[2]), + IceSedData(isds_baseline[3]), }; - // Get data from fortran - for (auto& d : isds_fortran) { - ice_sedimentation(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < num_runs; ++i) { + isds_baseline[i].read(Base::m_fid); + } } // Get data from cxx for (auto& d : isds_cxx) { - ice_sedimentation_f(d.kts, d.kte, d.ktop, d.kbot, d.kdir, + ice_sedimentation_host(d.kts, d.kte, d.ktop, d.kbot, d.kdir, d.rho, d.inv_rho, d.rhofaci, d.cld_frac_i, d.inv_dz, d.dt, d.inv_dt, d.qi, d.qi_incld, d.ni, d.qm, d.qm_incld, d.bm, d.bm_incld, d.ni_incld, &d.precip_ice_surf, d.qi_tend, d.ni_tend); } - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { // Due to pack issues, we must restrict checks to the active k space - Int start = std::min(isds_fortran[i].kbot, isds_fortran[i].ktop) - 1; // 0-based indx - Int end = std::max(isds_fortran[i].kbot, isds_fortran[i].ktop); // 0-based indx + Int start = std::min(isds_baseline[i].kbot, isds_baseline[i].ktop) - 1; // 0-based indx + Int end = std::max(isds_baseline[i].kbot, isds_baseline[i].ktop); // 0-based indx for (Int k = start; k < end; ++k) { - REQUIRE(isds_fortran[i].qi[k] == isds_cxx[i].qi[k]); - REQUIRE(isds_fortran[i].qi_incld[k] == isds_cxx[i].qi_incld[k]); - REQUIRE(isds_fortran[i].ni[k] == isds_cxx[i].ni[k]); - REQUIRE(isds_fortran[i].ni_incld[k] == isds_cxx[i].ni_incld[k]); - REQUIRE(isds_fortran[i].qm[k] == isds_cxx[i].qm[k]); - REQUIRE(isds_fortran[i].qm_incld[k] == isds_cxx[i].qm_incld[k]); - REQUIRE(isds_fortran[i].bm[k] == isds_cxx[i].bm[k]); - REQUIRE(isds_fortran[i].bm_incld[k] == isds_cxx[i].bm_incld[k]); - REQUIRE(isds_fortran[i].qi_tend[k] == isds_cxx[i].qi_tend[k]); - REQUIRE(isds_fortran[i].ni_tend[k] == isds_cxx[i].ni_tend[k]); + REQUIRE(isds_baseline[i].qi[k] == isds_cxx[i].qi[k]); + REQUIRE(isds_baseline[i].qi_incld[k] == isds_cxx[i].qi_incld[k]); + REQUIRE(isds_baseline[i].ni[k] == isds_cxx[i].ni[k]); + REQUIRE(isds_baseline[i].ni_incld[k] == isds_cxx[i].ni_incld[k]); + REQUIRE(isds_baseline[i].qm[k] == isds_cxx[i].qm[k]); + REQUIRE(isds_baseline[i].qm_incld[k] == isds_cxx[i].qm_incld[k]); + REQUIRE(isds_baseline[i].bm[k] == isds_cxx[i].bm[k]); + REQUIRE(isds_baseline[i].bm_incld[k] == isds_cxx[i].bm_incld[k]); + REQUIRE(isds_baseline[i].qi_tend[k] == isds_cxx[i].qi_tend[k]); + REQUIRE(isds_baseline[i].ni_tend[k] == isds_cxx[i].ni_tend[k]); } - REQUIRE(isds_fortran[i].precip_ice_surf == isds_cxx[i].precip_ice_surf); + REQUIRE(isds_baseline[i].precip_ice_surf == isds_cxx[i].precip_ice_surf); + } + } + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + isds_cxx[i].write(Base::m_fid); } } } -static void run_bfb_homogeneous_freezing() +void run_bfb_homogeneous_freezing() { constexpr Scalar latice = C::LatIce; - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - HomogeneousFreezingData hfds_fortran[] = { + HomogeneousFreezingData hfds_baseline[] = { // kts, kte, ktop, kbot, kdir HomogeneousFreezingData(1, 72, 27, 72, -1), HomogeneousFreezingData(1, 72, 72, 27, 1), @@ -199,10 +212,10 @@ static void run_bfb_homogeneous_freezing() HomogeneousFreezingData(1, 72, 27, 27, 1), }; - static constexpr Int num_runs = sizeof(hfds_fortran) / sizeof(HomogeneousFreezingData); + static constexpr Int num_runs = sizeof(hfds_baseline) / sizeof(HomogeneousFreezingData); // Set up random input data - for (auto& d : hfds_fortran) { + for (auto& d : hfds_baseline) { const auto qsmall_r = std::make_pair(C::QSMALL/2, C::QSMALL*2); d.randomize(engine, { {d.T_atm, {C::T_homogfrz - 10, C::T_homogfrz + 10}}, {d.qc, qsmall_r}, {d.qr, qsmall_r} }); @@ -213,48 +226,56 @@ static void run_bfb_homogeneous_freezing() } } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state HomogeneousFreezingData hfds_cxx[num_runs] = { - HomogeneousFreezingData(hfds_fortran[0]), - HomogeneousFreezingData(hfds_fortran[1]), - HomogeneousFreezingData(hfds_fortran[2]), - HomogeneousFreezingData(hfds_fortran[3]), + HomogeneousFreezingData(hfds_baseline[0]), + HomogeneousFreezingData(hfds_baseline[1]), + HomogeneousFreezingData(hfds_baseline[2]), + HomogeneousFreezingData(hfds_baseline[3]), }; - // Get data from fortran - for (auto& d : hfds_fortran) { - homogeneous_freezing(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : hfds_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : hfds_cxx) { - homogeneous_freezing_f(d.kts, d.kte, d.ktop, d.kbot, d.kdir, + homogeneous_freezing_host(d.kts, d.kte, d.ktop, d.kbot, d.kdir, d.T_atm, d.inv_exner, d.qc, d.nc, d.qr, d.nr, d.qi, d.ni, d.qm, d.bm, d.th_atm); } - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { // Due to pack issues, we must restrict checks to the active k space - Int start = std::min(hfds_fortran[i].kbot, hfds_fortran[i].ktop) - 1; // 0-based indx - Int end = std::max(hfds_fortran[i].kbot, hfds_fortran[i].ktop); // 0-based indx + Int start = std::min(hfds_baseline[i].kbot, hfds_baseline[i].ktop) - 1; // 0-based indx + Int end = std::max(hfds_baseline[i].kbot, hfds_baseline[i].ktop); // 0-based indx for (Int k = start; k < end; ++k) { - REQUIRE(hfds_fortran[i].qc[k] == hfds_cxx[i].qc[k]); - REQUIRE(hfds_fortran[i].nc[k] == hfds_cxx[i].nc[k]); - REQUIRE(hfds_fortran[i].qr[k] == hfds_cxx[i].qr[k]); - REQUIRE(hfds_fortran[i].nr[k] == hfds_cxx[i].nr[k]); - REQUIRE(hfds_fortran[i].qi[k] == hfds_cxx[i].qi[k]); - REQUIRE(hfds_fortran[i].ni[k] == hfds_cxx[i].ni[k]); - REQUIRE(hfds_fortran[i].qm[k] == hfds_cxx[i].qm[k]); - REQUIRE(hfds_fortran[i].bm[k] == hfds_cxx[i].bm[k]); - REQUIRE(hfds_fortran[i].th_atm[k] == hfds_cxx[i].th_atm[k]); + REQUIRE(hfds_baseline[i].qc[k] == hfds_cxx[i].qc[k]); + REQUIRE(hfds_baseline[i].nc[k] == hfds_cxx[i].nc[k]); + REQUIRE(hfds_baseline[i].qr[k] == hfds_cxx[i].qr[k]); + REQUIRE(hfds_baseline[i].nr[k] == hfds_cxx[i].nr[k]); + REQUIRE(hfds_baseline[i].qi[k] == hfds_cxx[i].qi[k]); + REQUIRE(hfds_baseline[i].ni[k] == hfds_cxx[i].ni[k]); + REQUIRE(hfds_baseline[i].qm[k] == hfds_cxx[i].qm[k]); + REQUIRE(hfds_baseline[i].bm[k] == hfds_cxx[i].bm[k]); + REQUIRE(hfds_baseline[i].th_atm[k] == hfds_cxx[i].th_atm[k]); } } } + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + hfds_cxx[i].write(Base::m_fid); + } + } + } -static void run_bfb() +void run_bfb() { run_bfb_calc_bulk_rhime(); run_bfb_ice_sed(); @@ -271,12 +292,11 @@ namespace { TEST_CASE("p3_ice_sed", "[p3_functions]") { - using TCS = scream::p3::unit_test::UnitWrap::UnitTest::TestIceSed; - - TCS::run_phys(); - TCS::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIceSed; - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp index 5873b8335c38..dcbdfc618263 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp @@ -4,8 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" -#include "share/util/scream_setup_random_test.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -14,38 +13,40 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestIceSupersatConservation { +struct UnitWrap::UnitTest::TestIceSupersatConservation : public UnitWrap::UnitTest::Base { - static void run_bfb() + void run_bfb() { constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - IceSupersatConservationData f90_data[max_pack_size]; + IceSupersatConservationData baseline_data[max_pack_size]; // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { + // Alternatively, you can use the baseline_data construtors/initializer lists to hardcode data + for (auto& d : baseline_data) { d.randomize(engine); - d.dt = f90_data[0].dt; // hold this fixed, it is not packed data + d.dt = baseline_data[0].dt; // hold this fixed, it is not packed data // C++ impl uses constants for latent_heat values. Manually set here // so F90 can match d.latent_heat_sublim = latvap+latice; } - // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that + // Create copies of data for use by cxx and sync it to device. Needs to happen before reads so that // inout data is in original state view_1d cxx_device("cxx_device", max_pack_size); const auto cxx_host = Kokkos::create_mirror_view(cxx_device); - std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); + std::copy(&baseline_data[0], &baseline_data[0] + max_pack_size, cxx_host.data()); Kokkos::deep_copy(cxx_device, cxx_host); - // Get data from fortran - for (auto& d : f90_data) { - ice_supersat_conservation(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + baseline_data[i].read(Base::m_fid); + } } // Get data from cxx. Run ice_supersat_conservation from a kernel and copy results back to host @@ -72,20 +73,24 @@ struct UnitWrap::UnitTest::TestIceSupersatConservation { cxx_device(vs).qidep = qidep[s]; cxx_device(vs).qinuc = qinuc[s]; } - }); Kokkos::deep_copy(cxx_host, cxx_device); // Verify BFB results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - IceSupersatConservationData& d_f90 = f90_data[i]; + IceSupersatConservationData& d_f90 = baseline_data[i]; IceSupersatConservationData& d_cxx = cxx_host[i]; REQUIRE(d_f90.qidep == d_cxx.qidep); REQUIRE(d_f90.qinuc == d_cxx.qinuc); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + cxx_host(s).write(Base::m_fid); + } + } } // run_bfb }; @@ -98,9 +103,10 @@ namespace { TEST_CASE("ice_supersat_conservation_bfb", "[p3]") { - using TestStruct = scream::p3::unit_test::UnitWrap::UnitTest::TestIceSupersatConservation; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIceSupersatConservation; - TestStruct::run_bfb(); + T t; + t.run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp index 091d05895fcc..0c75d81ab6e6 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -22,47 +22,10 @@ namespace unit_test { */ template -struct UnitWrap::UnitTest::TestTableIce { - - static void test_read_lookup_tables_bfb() - { - // Read in ice tables - view_ice_table ice_table_vals; - view_collect_table collect_table_vals; - Functions::init_kokkos_ice_lookup_tables(ice_table_vals, collect_table_vals); - - // Get data from fortran - P3InitAFortranData d; - p3_init_a(d); - - // Copy device data to host - const auto ice_table_vals_host = Kokkos::create_mirror_view(ice_table_vals); - const auto collect_table_vals_host = Kokkos::create_mirror_view(collect_table_vals); - Kokkos::deep_copy(ice_table_vals_host, ice_table_vals); - Kokkos::deep_copy(collect_table_vals_host, collect_table_vals); - - // Compare (on host) - for (size_t i = 0; i < ice_table_vals_host.extent(0); ++i) { - for (size_t j = 0; j < ice_table_vals_host.extent(1); ++j) { - for (size_t k = 0; k < ice_table_vals_host.extent(2); ++k) { - - for (size_t l = 0; l < ice_table_vals_host.extent(3); ++l) { - REQUIRE(ice_table_vals_host(i, j, k, l) == d.ice_table_vals(i, j, k, l)); - } - - for (size_t l = 0; l < collect_table_vals_host.extent(3); ++l) { - for (size_t m = 0; m < collect_table_vals_host.extent(4); ++m) { - REQUIRE(collect_table_vals_host(i, j, k, l, m) == d.collect_table_vals(i, j, k, l, m)); - } - } - - } - } - } - } +struct UnitWrap::UnitTest::TestTableIce : public UnitWrap::UnitTest::Base { template - static void init_table_linear_dimension(View& table, int linear_dimension) + void init_table_linear_dimension(View& table, int linear_dimension) { // set up views using NonConstView = typename View::non_const_type; @@ -96,14 +59,14 @@ struct UnitWrap::UnitTest::TestTableIce { table = view_device; } - static void run_bfb() + void run_bfb() { using KTH = KokkosTypes; // Read in ice tables view_ice_table ice_table_vals; view_collect_table collect_table_vals; - Functions::init_kokkos_ice_lookup_tables(ice_table_vals, collect_table_vals); + Functions::get_global_ice_lookup_tables(ice_table_vals, collect_table_vals); constexpr Scalar qsmall = C::QSMALL; @@ -199,14 +162,6 @@ struct UnitWrap::UnitTest::TestTableIce { {lid[15], lidb[15], access_table_index} }; - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - find_lookuptable_indices_1a(lid[i]); - find_lookuptable_indices_1b(lidb[i]); - access_lookup_table(altd[i]); - access_lookup_table_coll(altcd[i]); - } - // Sync to device KTH::view_1d lid_host("lid_host", max_pack_size); KTH::view_1d lidb_host("lidb_host", max_pack_size); @@ -217,6 +172,16 @@ struct UnitWrap::UnitTest::TestTableIce { Kokkos::deep_copy(lid_device, lid_host); Kokkos::deep_copy(lidb_device, lidb_host); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + lid[i].read(Base::m_fid); + lidb[i].read(Base::m_fid); + altd[i].read(Base::m_fid); + altcd[i].read(Base::m_fid); + } + } + // Run the lookup from a kernel and copy results back to host view_2d int_results("int results", 5, max_pack_size); view_2d real_results("real results", 7, max_pack_size); @@ -270,15 +235,14 @@ struct UnitWrap::UnitTest::TestTableIce { Kokkos::deep_copy(real_results_mirror, real_results); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for(int s = 0; s < max_pack_size; ++s) { - // +1 for O vs 1-based indexing - REQUIRE(int_results_mirror(0, s)+1 == lid[s].dumi); - REQUIRE(int_results_mirror(1, s)+1 == lid[s].dumjj); - REQUIRE(int_results_mirror(2, s)+1 == lid[s].dumii); - REQUIRE(int_results_mirror(3, s)+1 == lid[s].dumzz); + REQUIRE(int_results_mirror(0, s) == lid[s].dumi); + REQUIRE(int_results_mirror(1, s) == lid[s].dumjj); + REQUIRE(int_results_mirror(2, s) == lid[s].dumii); + REQUIRE(int_results_mirror(3, s) == lid[s].dumzz); - REQUIRE(int_results_mirror(4, s)+1 == lidb[s].dumj); + REQUIRE(int_results_mirror(4, s) == lidb[s].dumj); REQUIRE(real_results_mirror(0, s) == lid[s].dum1); REQUIRE(real_results_mirror(1, s) == lid[s].dum4); @@ -292,9 +256,35 @@ struct UnitWrap::UnitTest::TestTableIce { REQUIRE(real_results_mirror(6, s) == altcd[s].proc); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + lid[s].dumi = int_results_mirror(0, s); + lid[s].dumjj = int_results_mirror(1, s); + lid[s].dumii = int_results_mirror(2, s); + lid[s].dumzz = int_results_mirror(3, s); + + lidb[s].dumj = int_results_mirror(4, s); + + lid[s].dum1 = real_results_mirror(0, s); + lid[s].dum4 = real_results_mirror(1, s); + lid[s].dum5 = real_results_mirror(2, s); + lid[s].dum6 = real_results_mirror(3, s); + + lidb[s].dum3 = real_results_mirror(4, s); + + altd[s].proc = real_results_mirror(5, s); + + altcd[s].proc = real_results_mirror(6, s); + + lid[s].write(Base::m_fid); + lidb[s].write(Base::m_fid); + altd[s].write(Base::m_fid); + altcd[s].write(Base::m_fid); + } + } } - static void run_phys() + void run_phys() { #if 0 view_ice_table ice_table_vals; @@ -343,11 +333,11 @@ namespace { TEST_CASE("p3_ice_tables", "[p3_functions]") { - using TTI = scream::p3::unit_test::UnitWrap::UnitTest::TestTableIce; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestTableIce; - TTI::test_read_lookup_tables_bfb(); - TTI::run_phys(); - TTI::run_bfb(); + T t; + t.run_phys(); + t.run_bfb(); } } diff --git a/components/eamxx/src/physics/p3/tests/p3_incloud_mixingratios_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_incloud_mixingratios_unit_tests.cpp index 4a405fcc7c43..eae1454c36a0 100644 --- a/components/eamxx/src/physics/p3/tests/p3_incloud_mixingratios_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_incloud_mixingratios_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -19,9 +19,9 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestIncloudMixing { +struct UnitWrap::UnitTest::TestIncloudMixing : public UnitWrap::UnitTest::Base { - static void run_incloud_mixing_bfb() + void run_incloud_mixing_bfb() { using KTH = KokkosTypes; @@ -69,9 +69,11 @@ struct UnitWrap::UnitTest::TestIncloudMixing { std::copy(&self[0], &self[0] + max_pack_size, self_host.data()); Kokkos::deep_copy(self_device, self_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - calculate_incloud_mixingratios(self[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + self[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -115,7 +117,7 @@ struct UnitWrap::UnitTest::TestIncloudMixing { Kokkos::deep_copy(self_host, self_device); - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(self[s].qc_incld == self_host(s).qc_incld); REQUIRE(self[s].qr_incld == self_host(s).qr_incld); @@ -127,9 +129,14 @@ struct UnitWrap::UnitTest::TestIncloudMixing { REQUIRE(self[s].bm_incld == self_host(s).bm_incld); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + self_host(s).write(Base::m_fid); + } + } } - static void run_incloud_mixing_phys() + void run_incloud_mixing_phys() { // TODO } @@ -143,10 +150,11 @@ namespace { TEST_CASE("p3_incloud_mixingratios", "[p3_functions]") { - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestIncloudMixing; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIncloudMixing; - TD::run_incloud_mixing_phys(); - TD::run_incloud_mixing_bfb(); + T t; + t.run_incloud_mixing_phys(); + t.run_incloud_mixing_bfb(); } } diff --git a/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp index a804bd8756d5..c60efd2b8824 100644 --- a/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp @@ -4,8 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" -#include "share/util/scream_setup_random_test.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -19,29 +18,29 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestP3Main { +struct UnitWrap::UnitTest::TestP3Main : public UnitWrap::UnitTest::Base { -static void run_phys_p3_main_part1() +void run_phys_p3_main_part1() { // TODO } -static void run_phys_p3_main_part2() +void run_phys_p3_main_part2() { // TODO } -static void run_phys_p3_main_part3() +void run_phys_p3_main_part3() { // TODO } -static void run_phys_p3_main() +void run_phys_p3_main() { // TODO } -static void run_phys() +void run_phys() { run_phys_p3_main_part1(); run_phys_p3_main_part2(); @@ -49,9 +48,9 @@ static void run_phys() run_phys_p3_main(); } -static void run_bfb_p3_main_part1() +void run_bfb_p3_main_part1() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); constexpr Scalar qsmall = C::QSMALL; //PMC wouldn't it make more sense to define qsmall at a higher level since used in part1, part2, and part3? constexpr Scalar T_zerodegc = C::T_zerodegc; @@ -60,7 +59,7 @@ static void run_bfb_p3_main_part1() constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; - P3MainPart1Data isds_fortran[] = { + P3MainPart1Data isds_baseline[] = { // kts, kte, ktop, kbot, kdir, do_predict_nc, do_prescribed_CCN, dt P3MainPart1Data(1, 72, 1, 72, 1, false, true, 1.800E+03), P3MainPart1Data(1, 72, 1, 72, 1, true, true, 1.800E+03), @@ -68,9 +67,9 @@ static void run_bfb_p3_main_part1() P3MainPart1Data(1, 72, 72, 1, -1, true, false, 1.800E+03), }; - static constexpr Int num_runs = sizeof(isds_fortran) / sizeof(P3MainPart1Data); + static constexpr Int num_runs = sizeof(isds_baseline) / sizeof(P3MainPart1Data); - for (auto& d : isds_fortran) { + for (auto& d : isds_baseline) { const auto qsmall_r = std::make_pair(0, qsmall*2); //PMC this range seems inappropriately small d.randomize(engine, { {d.T_atm, {T_zerodegc - 10, T_zerodegc + 10}}, @@ -86,23 +85,25 @@ static void run_bfb_p3_main_part1() } } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state P3MainPart1Data isds_cxx[num_runs] = { - P3MainPart1Data(isds_fortran[0]), - P3MainPart1Data(isds_fortran[1]), - P3MainPart1Data(isds_fortran[2]), - P3MainPart1Data(isds_fortran[3]), + P3MainPart1Data(isds_baseline[0]), + P3MainPart1Data(isds_baseline[1]), + P3MainPart1Data(isds_baseline[2]), + P3MainPart1Data(isds_baseline[3]), }; - // Get data from fortran - for (auto& d : isds_fortran) { - p3_main_part1(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : isds_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : isds_cxx) { - p3_main_part1_f(d.kts, d.kte, d.ktop, d.kbot, d.kdir, d.do_predict_nc, d.do_prescribed_CCN, d.dt, + p3_main_part1_host(d.kts, d.kte, d.ktop, d.kbot, d.kdir, d.do_predict_nc, d.do_prescribed_CCN, d.dt, d.pres, d.dpres, d.dz, d.nc_nuceat_tend, d.nccn_prescribed, d.inv_exner, d.exner, d.inv_cld_frac_l, d.inv_cld_frac_i, d.inv_cld_frac_r, d.T_atm, d.rho, d.inv_rho, d.qv_sat_l, d.qv_sat_i, d.qv_supersat_i, d.rhofacr, d.rhofaci, @@ -111,48 +112,53 @@ static void run_bfb_p3_main_part1() &d.is_nucleat_possible, &d.is_hydromet_present); } - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - Int start = std::min(isds_fortran[i].kbot, isds_fortran[i].ktop) - 1; // 0-based indx - Int end = std::max(isds_fortran[i].kbot, isds_fortran[i].ktop); // 0-based indx + Int start = std::min(isds_baseline[i].kbot, isds_baseline[i].ktop) - 1; // 0-based indx + Int end = std::max(isds_baseline[i].kbot, isds_baseline[i].ktop); // 0-based indx for (Int k = start; k < end; ++k) { - REQUIRE(isds_fortran[i].T_atm[k] == isds_cxx[i].T_atm[k]); - REQUIRE(isds_fortran[i].rho[k] == isds_cxx[i].rho[k]); - REQUIRE(isds_fortran[i].inv_rho[k] == isds_cxx[i].inv_rho[k]); - REQUIRE(isds_fortran[i].qv_sat_l[k] == isds_cxx[i].qv_sat_l[k]); - REQUIRE(isds_fortran[i].qv_sat_i[k] == isds_cxx[i].qv_sat_i[k]); - REQUIRE(isds_fortran[i].qv_supersat_i[k] == isds_cxx[i].qv_supersat_i[k]); - REQUIRE(isds_fortran[i].rhofacr[k] == isds_cxx[i].rhofacr[k]); - REQUIRE(isds_fortran[i].rhofaci[k] == isds_cxx[i].rhofaci[k]); - REQUIRE(isds_fortran[i].acn[k] == isds_cxx[i].acn[k]); - REQUIRE(isds_fortran[i].qv[k] == isds_cxx[i].qv[k]); - REQUIRE(isds_fortran[i].th_atm[k] == isds_cxx[i].th_atm[k]); - REQUIRE(isds_fortran[i].qc[k] == isds_cxx[i].qc[k]); - REQUIRE(isds_fortran[i].nc[k] == isds_cxx[i].nc[k]); - REQUIRE(isds_fortran[i].qr[k] == isds_cxx[i].qr[k]); - REQUIRE(isds_fortran[i].nr[k] == isds_cxx[i].nr[k]); - REQUIRE(isds_fortran[i].qi[k] == isds_cxx[i].qi[k]); - REQUIRE(isds_fortran[i].ni[k] == isds_cxx[i].ni[k]); - REQUIRE(isds_fortran[i].qm[k] == isds_cxx[i].qm[k]); - REQUIRE(isds_fortran[i].bm[k] == isds_cxx[i].bm[k]); - REQUIRE(isds_fortran[i].qc_incld[k] == isds_cxx[i].qc_incld[k]); - REQUIRE(isds_fortran[i].qr_incld[k] == isds_cxx[i].qr_incld[k]); - REQUIRE(isds_fortran[i].qi_incld[k] == isds_cxx[i].qi_incld[k]); - REQUIRE(isds_fortran[i].qm_incld[k] == isds_cxx[i].qm_incld[k]); - REQUIRE(isds_fortran[i].nc_incld[k] == isds_cxx[i].nc_incld[k]); - REQUIRE(isds_fortran[i].nr_incld[k] == isds_cxx[i].nr_incld[k]); - REQUIRE(isds_fortran[i].ni_incld[k] == isds_cxx[i].ni_incld[k]); - REQUIRE(isds_fortran[i].bm_incld[k] == isds_cxx[i].bm_incld[k]); + REQUIRE(isds_baseline[i].T_atm[k] == isds_cxx[i].T_atm[k]); + REQUIRE(isds_baseline[i].rho[k] == isds_cxx[i].rho[k]); + REQUIRE(isds_baseline[i].inv_rho[k] == isds_cxx[i].inv_rho[k]); + REQUIRE(isds_baseline[i].qv_sat_l[k] == isds_cxx[i].qv_sat_l[k]); + REQUIRE(isds_baseline[i].qv_sat_i[k] == isds_cxx[i].qv_sat_i[k]); + REQUIRE(isds_baseline[i].qv_supersat_i[k] == isds_cxx[i].qv_supersat_i[k]); + REQUIRE(isds_baseline[i].rhofacr[k] == isds_cxx[i].rhofacr[k]); + REQUIRE(isds_baseline[i].rhofaci[k] == isds_cxx[i].rhofaci[k]); + REQUIRE(isds_baseline[i].acn[k] == isds_cxx[i].acn[k]); + REQUIRE(isds_baseline[i].qv[k] == isds_cxx[i].qv[k]); + REQUIRE(isds_baseline[i].th_atm[k] == isds_cxx[i].th_atm[k]); + REQUIRE(isds_baseline[i].qc[k] == isds_cxx[i].qc[k]); + REQUIRE(isds_baseline[i].nc[k] == isds_cxx[i].nc[k]); + REQUIRE(isds_baseline[i].qr[k] == isds_cxx[i].qr[k]); + REQUIRE(isds_baseline[i].nr[k] == isds_cxx[i].nr[k]); + REQUIRE(isds_baseline[i].qi[k] == isds_cxx[i].qi[k]); + REQUIRE(isds_baseline[i].ni[k] == isds_cxx[i].ni[k]); + REQUIRE(isds_baseline[i].qm[k] == isds_cxx[i].qm[k]); + REQUIRE(isds_baseline[i].bm[k] == isds_cxx[i].bm[k]); + REQUIRE(isds_baseline[i].qc_incld[k] == isds_cxx[i].qc_incld[k]); + REQUIRE(isds_baseline[i].qr_incld[k] == isds_cxx[i].qr_incld[k]); + REQUIRE(isds_baseline[i].qi_incld[k] == isds_cxx[i].qi_incld[k]); + REQUIRE(isds_baseline[i].qm_incld[k] == isds_cxx[i].qm_incld[k]); + REQUIRE(isds_baseline[i].nc_incld[k] == isds_cxx[i].nc_incld[k]); + REQUIRE(isds_baseline[i].nr_incld[k] == isds_cxx[i].nr_incld[k]); + REQUIRE(isds_baseline[i].ni_incld[k] == isds_cxx[i].ni_incld[k]); + REQUIRE(isds_baseline[i].bm_incld[k] == isds_cxx[i].bm_incld[k]); } - REQUIRE( isds_fortran[i].is_hydromet_present == isds_cxx[i].is_hydromet_present ); - REQUIRE( isds_fortran[i].is_nucleat_possible == isds_cxx[i].is_nucleat_possible ); + REQUIRE( isds_baseline[i].is_hydromet_present == isds_cxx[i].is_hydromet_present ); + REQUIRE( isds_baseline[i].is_nucleat_possible == isds_cxx[i].is_nucleat_possible ); + } + } + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + isds_cxx[i].write(Base::m_fid); } } } -static void run_bfb_p3_main_part2() +void run_bfb_p3_main_part2() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); constexpr Scalar qsmall = C::QSMALL; constexpr Scalar T_zerodegc = C::T_zerodegc; @@ -161,7 +167,7 @@ static void run_bfb_p3_main_part2() constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; - P3MainPart2Data isds_fortran[] = { + P3MainPart2Data isds_baseline[] = { // kts, kte, ktop, kbot, kdir, do_predict_nc, do_prescribed_CCN, dt P3MainPart2Data(1, 72, 1, 72, 1, false, true, 1.800E+03), P3MainPart2Data(1, 72, 1, 72, 1, true, true, 1.800E+03), @@ -169,9 +175,9 @@ static void run_bfb_p3_main_part2() P3MainPart2Data(1, 72, 72, 1, -1, true, false, 1.800E+03), }; - static constexpr Int num_runs = sizeof(isds_fortran) / sizeof(P3MainPart2Data); + static constexpr Int num_runs = sizeof(isds_baseline) / sizeof(P3MainPart2Data); - for (auto& d : isds_fortran) { + for (auto& d : isds_baseline) { const auto qsmall_r = std::make_pair(0, qsmall*2); d.randomize(engine, { {d.T_atm, {T_zerodegc - 10, T_zerodegc + 10}}, @@ -188,23 +194,25 @@ static void run_bfb_p3_main_part2() } } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state P3MainPart2Data isds_cxx[num_runs] = { - P3MainPart2Data(isds_fortran[0]), - P3MainPart2Data(isds_fortran[1]), - P3MainPart2Data(isds_fortran[2]), - P3MainPart2Data(isds_fortran[3]), + P3MainPart2Data(isds_baseline[0]), + P3MainPart2Data(isds_baseline[1]), + P3MainPart2Data(isds_baseline[2]), + P3MainPart2Data(isds_baseline[3]), }; - // Get data from fortran - for (auto& d : isds_fortran) { - p3_main_part2(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : isds_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : isds_cxx) { - p3_main_part2_f( + p3_main_part2_host( d.kts, d.kte, d.kbot, d.ktop, d.kdir, d.do_predict_nc, d.do_prescribed_CCN, d.dt, d.inv_dt, d.pres, d.dpres, d.dz, d.nc_nuceat_tend, d.inv_exner, d.exner, d.inv_cld_frac_l, d.inv_cld_frac_i, d.inv_cld_frac_r, d.ni_activated, d.inv_qc_relvar, d.cld_frac_i, d.cld_frac_l, d.cld_frac_r, d.qv_prev, d.t_prev, @@ -215,75 +223,80 @@ static void run_bfb_p3_main_part2() d.prctot, &d.is_hydromet_present); } - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - Int start = std::min(isds_fortran[i].kbot, isds_fortran[i].ktop) - 1; // 0-based indx - Int end = std::max(isds_fortran[i].kbot, isds_fortran[i].ktop); // 0-based indx + Int start = std::min(isds_baseline[i].kbot, isds_baseline[i].ktop) - 1; // 0-based indx + Int end = std::max(isds_baseline[i].kbot, isds_baseline[i].ktop); // 0-based indx for (Int k = start; k < end; ++k) { - REQUIRE(isds_fortran[i].T_atm[k] == isds_cxx[i].T_atm[k]); - REQUIRE(isds_fortran[i].rho[k] == isds_cxx[i].rho[k]); - REQUIRE(isds_fortran[i].inv_rho[k] == isds_cxx[i].inv_rho[k]); - REQUIRE(isds_fortran[i].qv_sat_l[k] == isds_cxx[i].qv_sat_l[k]); - REQUIRE(isds_fortran[i].qv_sat_i[k] == isds_cxx[i].qv_sat_i[k]); - REQUIRE(isds_fortran[i].qv_supersat_i[k] == isds_cxx[i].qv_supersat_i[k]); - REQUIRE(isds_fortran[i].rhofacr[k] == isds_cxx[i].rhofacr[k]); - REQUIRE(isds_fortran[i].rhofaci[k] == isds_cxx[i].rhofaci[k]); - REQUIRE(isds_fortran[i].acn[k] == isds_cxx[i].acn[k]); - REQUIRE(isds_fortran[i].qv[k] == isds_cxx[i].qv[k]); - REQUIRE(isds_fortran[i].th_atm[k] == isds_cxx[i].th_atm[k]); - REQUIRE(isds_fortran[i].qc[k] == isds_cxx[i].qc[k]); - REQUIRE(isds_fortran[i].nc[k] == isds_cxx[i].nc[k]); - REQUIRE(isds_fortran[i].qr[k] == isds_cxx[i].qr[k]); - REQUIRE(isds_fortran[i].nr[k] == isds_cxx[i].nr[k]); - REQUIRE(isds_fortran[i].qi[k] == isds_cxx[i].qi[k]); - REQUIRE(isds_fortran[i].ni[k] == isds_cxx[i].ni[k]); - REQUIRE(isds_fortran[i].qm[k] == isds_cxx[i].qm[k]); - REQUIRE(isds_fortran[i].bm[k] == isds_cxx[i].bm[k]); - REQUIRE(isds_fortran[i].latent_heat_vapor[k] == latvap); - REQUIRE(isds_fortran[i].latent_heat_sublim[k] == (latvap+latice)); - REQUIRE(isds_fortran[i].latent_heat_fusion[k] == latice); - REQUIRE(isds_fortran[i].qc_incld[k] == isds_cxx[i].qc_incld[k]); - REQUIRE(isds_fortran[i].qr_incld[k] == isds_cxx[i].qr_incld[k]); - REQUIRE(isds_fortran[i].qi_incld[k] == isds_cxx[i].qi_incld[k]); - REQUIRE(isds_fortran[i].qm_incld[k] == isds_cxx[i].qm_incld[k]); - REQUIRE(isds_fortran[i].nc_incld[k] == isds_cxx[i].nc_incld[k]); - REQUIRE(isds_fortran[i].nr_incld[k] == isds_cxx[i].nr_incld[k]); - REQUIRE(isds_fortran[i].ni_incld[k] == isds_cxx[i].ni_incld[k]); - REQUIRE(isds_fortran[i].bm_incld[k] == isds_cxx[i].bm_incld[k]); - REQUIRE(isds_fortran[i].mu_c[k] == isds_cxx[i].mu_c[k]); - REQUIRE(isds_fortran[i].nu[k] == isds_cxx[i].nu[k]); - REQUIRE(isds_fortran[i].lamc[k] == isds_cxx[i].lamc[k]); - REQUIRE(isds_fortran[i].cdist[k] == isds_cxx[i].cdist[k]); - REQUIRE(isds_fortran[i].cdist1[k] == isds_cxx[i].cdist1[k]); - REQUIRE(isds_fortran[i].cdistr[k] == isds_cxx[i].cdistr[k]); - REQUIRE(isds_fortran[i].mu_r[k] == isds_cxx[i].mu_r[k]); - REQUIRE(isds_fortran[i].lamr[k] == isds_cxx[i].lamr[k]); - REQUIRE(isds_fortran[i].logn0r[k] == isds_cxx[i].logn0r[k]); - REQUIRE(isds_fortran[i].qv2qi_depos_tend[k] == isds_cxx[i].qv2qi_depos_tend[k]); - REQUIRE(isds_fortran[i].precip_total_tend[k] == isds_cxx[i].precip_total_tend[k]); - REQUIRE(isds_fortran[i].nevapr[k] == isds_cxx[i].nevapr[k]); - REQUIRE(isds_fortran[i].qr_evap_tend[k] == isds_cxx[i].qr_evap_tend[k]); - REQUIRE(isds_fortran[i].vap_liq_exchange[k] == isds_cxx[i].vap_liq_exchange[k]); - REQUIRE(isds_fortran[i].vap_ice_exchange[k] == isds_cxx[i].vap_ice_exchange[k]); - REQUIRE(isds_fortran[i].liq_ice_exchange[k] == isds_cxx[i].liq_ice_exchange[k]); - REQUIRE(isds_fortran[i].pratot[k] == isds_cxx[i].pratot[k]); - REQUIRE(isds_fortran[i].prctot[k] == isds_cxx[i].prctot[k]); + REQUIRE(isds_baseline[i].T_atm[k] == isds_cxx[i].T_atm[k]); + REQUIRE(isds_baseline[i].rho[k] == isds_cxx[i].rho[k]); + REQUIRE(isds_baseline[i].inv_rho[k] == isds_cxx[i].inv_rho[k]); + REQUIRE(isds_baseline[i].qv_sat_l[k] == isds_cxx[i].qv_sat_l[k]); + REQUIRE(isds_baseline[i].qv_sat_i[k] == isds_cxx[i].qv_sat_i[k]); + REQUIRE(isds_baseline[i].qv_supersat_i[k] == isds_cxx[i].qv_supersat_i[k]); + REQUIRE(isds_baseline[i].rhofacr[k] == isds_cxx[i].rhofacr[k]); + REQUIRE(isds_baseline[i].rhofaci[k] == isds_cxx[i].rhofaci[k]); + REQUIRE(isds_baseline[i].acn[k] == isds_cxx[i].acn[k]); + REQUIRE(isds_baseline[i].qv[k] == isds_cxx[i].qv[k]); + REQUIRE(isds_baseline[i].th_atm[k] == isds_cxx[i].th_atm[k]); + REQUIRE(isds_baseline[i].qc[k] == isds_cxx[i].qc[k]); + REQUIRE(isds_baseline[i].nc[k] == isds_cxx[i].nc[k]); + REQUIRE(isds_baseline[i].qr[k] == isds_cxx[i].qr[k]); + REQUIRE(isds_baseline[i].nr[k] == isds_cxx[i].nr[k]); + REQUIRE(isds_baseline[i].qi[k] == isds_cxx[i].qi[k]); + REQUIRE(isds_baseline[i].ni[k] == isds_cxx[i].ni[k]); + REQUIRE(isds_baseline[i].qm[k] == isds_cxx[i].qm[k]); + REQUIRE(isds_baseline[i].bm[k] == isds_cxx[i].bm[k]); + REQUIRE(isds_baseline[i].latent_heat_vapor[k] == latvap); + REQUIRE(isds_baseline[i].latent_heat_sublim[k] == (latvap+latice)); + REQUIRE(isds_baseline[i].latent_heat_fusion[k] == latice); + REQUIRE(isds_baseline[i].qc_incld[k] == isds_cxx[i].qc_incld[k]); + REQUIRE(isds_baseline[i].qr_incld[k] == isds_cxx[i].qr_incld[k]); + REQUIRE(isds_baseline[i].qi_incld[k] == isds_cxx[i].qi_incld[k]); + REQUIRE(isds_baseline[i].qm_incld[k] == isds_cxx[i].qm_incld[k]); + REQUIRE(isds_baseline[i].nc_incld[k] == isds_cxx[i].nc_incld[k]); + REQUIRE(isds_baseline[i].nr_incld[k] == isds_cxx[i].nr_incld[k]); + REQUIRE(isds_baseline[i].ni_incld[k] == isds_cxx[i].ni_incld[k]); + REQUIRE(isds_baseline[i].bm_incld[k] == isds_cxx[i].bm_incld[k]); + REQUIRE(isds_baseline[i].mu_c[k] == isds_cxx[i].mu_c[k]); + REQUIRE(isds_baseline[i].nu[k] == isds_cxx[i].nu[k]); + REQUIRE(isds_baseline[i].lamc[k] == isds_cxx[i].lamc[k]); + REQUIRE(isds_baseline[i].cdist[k] == isds_cxx[i].cdist[k]); + REQUIRE(isds_baseline[i].cdist1[k] == isds_cxx[i].cdist1[k]); + REQUIRE(isds_baseline[i].cdistr[k] == isds_cxx[i].cdistr[k]); + REQUIRE(isds_baseline[i].mu_r[k] == isds_cxx[i].mu_r[k]); + REQUIRE(isds_baseline[i].lamr[k] == isds_cxx[i].lamr[k]); + REQUIRE(isds_baseline[i].logn0r[k] == isds_cxx[i].logn0r[k]); + REQUIRE(isds_baseline[i].qv2qi_depos_tend[k] == isds_cxx[i].qv2qi_depos_tend[k]); + REQUIRE(isds_baseline[i].precip_total_tend[k] == isds_cxx[i].precip_total_tend[k]); + REQUIRE(isds_baseline[i].nevapr[k] == isds_cxx[i].nevapr[k]); + REQUIRE(isds_baseline[i].qr_evap_tend[k] == isds_cxx[i].qr_evap_tend[k]); + REQUIRE(isds_baseline[i].vap_liq_exchange[k] == isds_cxx[i].vap_liq_exchange[k]); + REQUIRE(isds_baseline[i].vap_ice_exchange[k] == isds_cxx[i].vap_ice_exchange[k]); + REQUIRE(isds_baseline[i].liq_ice_exchange[k] == isds_cxx[i].liq_ice_exchange[k]); + REQUIRE(isds_baseline[i].pratot[k] == isds_cxx[i].pratot[k]); + REQUIRE(isds_baseline[i].prctot[k] == isds_cxx[i].prctot[k]); } - REQUIRE( isds_fortran[i].is_hydromet_present == isds_cxx[i].is_hydromet_present ); + REQUIRE( isds_baseline[i].is_hydromet_present == isds_cxx[i].is_hydromet_present ); + } + } + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + isds_cxx[i].write(Base::m_fid); } } } -static void run_bfb_p3_main_part3() +void run_bfb_p3_main_part3() { constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; - auto engine = setup_random_test(); + auto engine = Base::get_engine(); constexpr Scalar qsmall = C::QSMALL; - P3MainPart3Data isds_fortran[] = { + P3MainPart3Data isds_baseline[] = { // kts, kte, ktop, kbot, kdir P3MainPart3Data(1, 72, 1, 72, 1), P3MainPart3Data(1, 72, 1, 72, 1), @@ -291,9 +304,9 @@ static void run_bfb_p3_main_part3() P3MainPart3Data(1, 72, 72, 1, -1), }; - static constexpr Int num_runs = sizeof(isds_fortran) / sizeof(P3MainPart3Data); + static constexpr Int num_runs = sizeof(isds_baseline) / sizeof(P3MainPart3Data); - for (auto& d : isds_fortran) { + for (auto& d : isds_baseline) { const auto qsmall_r = std::make_pair(0, qsmall*2); d.randomize(engine, { {d.qc, qsmall_r}, {d.qr, qsmall_r}, {d.qi, qsmall_r} }); @@ -305,23 +318,25 @@ static void run_bfb_p3_main_part3() } } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state P3MainPart3Data isds_cxx[num_runs] = { - P3MainPart3Data(isds_fortran[0]), - P3MainPart3Data(isds_fortran[1]), - P3MainPart3Data(isds_fortran[2]), - P3MainPart3Data(isds_fortran[3]), + P3MainPart3Data(isds_baseline[0]), + P3MainPart3Data(isds_baseline[1]), + P3MainPart3Data(isds_baseline[2]), + P3MainPart3Data(isds_baseline[3]), }; - // Get data from fortran - for (auto& d : isds_fortran) { - p3_main_part3(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : isds_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : isds_cxx) { - p3_main_part3_f( + p3_main_part3_host( d.kts, d.kte, d.kbot, d.ktop, d.kdir, d.inv_exner, d.cld_frac_l, d.cld_frac_r, d.cld_frac_i, d.rho, d.inv_rho, d.rhofaci, d.qv, d.th_atm, d.qc, d.nc, d.qr, d.nr, d.qi, d.ni, d.qm, d.bm, @@ -329,59 +344,64 @@ static void run_bfb_p3_main_part3() d. ze_rain, d.ze_ice, d.diag_vm_qi, d.diag_eff_radius_qi, d.diag_diam_qi, d.rho_qi, d.diag_equiv_reflectivity, d.diag_eff_radius_qc, d.diag_eff_radius_qr); } - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - Int start = std::min(isds_fortran[i].kbot, isds_fortran[i].ktop) - 1; // 0-based indx - Int end = std::max(isds_fortran[i].kbot, isds_fortran[i].ktop); // 0-based indx + Int start = std::min(isds_baseline[i].kbot, isds_baseline[i].ktop) - 1; // 0-based indx + Int end = std::max(isds_baseline[i].kbot, isds_baseline[i].ktop); // 0-based indx for (Int k = start; k < end; ++k) { - REQUIRE(isds_fortran[i].rho[k] == isds_cxx[i].rho[k]); - REQUIRE(isds_fortran[i].inv_rho[k] == isds_cxx[i].inv_rho[k]); - REQUIRE(isds_fortran[i].rhofaci[k] == isds_cxx[i].rhofaci[k]); - REQUIRE(isds_fortran[i].qv[k] == isds_cxx[i].qv[k]); - REQUIRE(isds_fortran[i].th_atm[k] == isds_cxx[i].th_atm[k]); - REQUIRE(isds_fortran[i].qc[k] == isds_cxx[i].qc[k]); - REQUIRE(isds_fortran[i].nc[k] == isds_cxx[i].nc[k]); - REQUIRE(isds_fortran[i].qr[k] == isds_cxx[i].qr[k]); - REQUIRE(isds_fortran[i].nr[k] == isds_cxx[i].nr[k]); - REQUIRE(isds_fortran[i].qi[k] == isds_cxx[i].qi[k]); - REQUIRE(isds_fortran[i].ni[k] == isds_cxx[i].ni[k]); - REQUIRE(isds_fortran[i].qm[k] == isds_cxx[i].qm[k]); - REQUIRE(isds_fortran[i].bm[k] == isds_cxx[i].bm[k]); - REQUIRE(isds_fortran[i].latent_heat_vapor[k] == latvap); - REQUIRE(isds_fortran[i].latent_heat_sublim[k] == latvap+latice); - REQUIRE(isds_fortran[i].mu_c[k] == isds_cxx[i].mu_c[k]); - REQUIRE(isds_fortran[i].nu[k] == isds_cxx[i].nu[k]); - REQUIRE(isds_fortran[i].lamc[k] == isds_cxx[i].lamc[k]); - REQUIRE(isds_fortran[i].mu_r[k] == isds_cxx[i].mu_r[k]); - REQUIRE(isds_fortran[i].lamr[k] == isds_cxx[i].lamr[k]); - REQUIRE(isds_fortran[i].vap_liq_exchange[k] == isds_cxx[i].vap_liq_exchange[k]); - REQUIRE(isds_fortran[i].ze_rain[k] == isds_cxx[i].ze_rain[k]); - REQUIRE(isds_fortran[i].ze_ice[k] == isds_cxx[i].ze_ice[k]); - REQUIRE(isds_fortran[i].diag_vm_qi[k] == isds_cxx[i].diag_vm_qi[k]); - REQUIRE(isds_fortran[i].diag_eff_radius_qi[k] == isds_cxx[i].diag_eff_radius_qi[k]); - REQUIRE(isds_fortran[i].diag_diam_qi[k] == isds_cxx[i].diag_diam_qi[k]); - REQUIRE(isds_fortran[i].rho_qi[k] == isds_cxx[i].rho_qi[k]); - REQUIRE(isds_fortran[i].diag_equiv_reflectivity[k] == isds_cxx[i].diag_equiv_reflectivity[k]); - REQUIRE(isds_fortran[i].diag_eff_radius_qc[k] == isds_cxx[i].diag_eff_radius_qc[k]); - REQUIRE(isds_fortran[i].diag_eff_radius_qr[k] == isds_cxx[i].diag_eff_radius_qr[k]); + REQUIRE(isds_baseline[i].rho[k] == isds_cxx[i].rho[k]); + REQUIRE(isds_baseline[i].inv_rho[k] == isds_cxx[i].inv_rho[k]); + REQUIRE(isds_baseline[i].rhofaci[k] == isds_cxx[i].rhofaci[k]); + REQUIRE(isds_baseline[i].qv[k] == isds_cxx[i].qv[k]); + REQUIRE(isds_baseline[i].th_atm[k] == isds_cxx[i].th_atm[k]); + REQUIRE(isds_baseline[i].qc[k] == isds_cxx[i].qc[k]); + REQUIRE(isds_baseline[i].nc[k] == isds_cxx[i].nc[k]); + REQUIRE(isds_baseline[i].qr[k] == isds_cxx[i].qr[k]); + REQUIRE(isds_baseline[i].nr[k] == isds_cxx[i].nr[k]); + REQUIRE(isds_baseline[i].qi[k] == isds_cxx[i].qi[k]); + REQUIRE(isds_baseline[i].ni[k] == isds_cxx[i].ni[k]); + REQUIRE(isds_baseline[i].qm[k] == isds_cxx[i].qm[k]); + REQUIRE(isds_baseline[i].bm[k] == isds_cxx[i].bm[k]); + REQUIRE(isds_baseline[i].latent_heat_vapor[k] == latvap); + REQUIRE(isds_baseline[i].latent_heat_sublim[k] == latvap+latice); + REQUIRE(isds_baseline[i].mu_c[k] == isds_cxx[i].mu_c[k]); + REQUIRE(isds_baseline[i].nu[k] == isds_cxx[i].nu[k]); + REQUIRE(isds_baseline[i].lamc[k] == isds_cxx[i].lamc[k]); + REQUIRE(isds_baseline[i].mu_r[k] == isds_cxx[i].mu_r[k]); + REQUIRE(isds_baseline[i].lamr[k] == isds_cxx[i].lamr[k]); + REQUIRE(isds_baseline[i].vap_liq_exchange[k] == isds_cxx[i].vap_liq_exchange[k]); + REQUIRE(isds_baseline[i].ze_rain[k] == isds_cxx[i].ze_rain[k]); + REQUIRE(isds_baseline[i].ze_ice[k] == isds_cxx[i].ze_ice[k]); + REQUIRE(isds_baseline[i].diag_vm_qi[k] == isds_cxx[i].diag_vm_qi[k]); + REQUIRE(isds_baseline[i].diag_eff_radius_qi[k] == isds_cxx[i].diag_eff_radius_qi[k]); + REQUIRE(isds_baseline[i].diag_diam_qi[k] == isds_cxx[i].diag_diam_qi[k]); + REQUIRE(isds_baseline[i].rho_qi[k] == isds_cxx[i].rho_qi[k]); + REQUIRE(isds_baseline[i].diag_equiv_reflectivity[k] == isds_cxx[i].diag_equiv_reflectivity[k]); + REQUIRE(isds_baseline[i].diag_eff_radius_qc[k] == isds_cxx[i].diag_eff_radius_qc[k]); + REQUIRE(isds_baseline[i].diag_eff_radius_qr[k] == isds_cxx[i].diag_eff_radius_qr[k]); } } } + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + isds_cxx[i].write(Base::m_fid); + } + } } -static void run_bfb_p3_main() +void run_bfb_p3_main() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - P3MainData isds_fortran[] = { + P3MainData isds_baseline[] = { // its, ite, kts, kte, it, dt, do_predict_nc, do_prescribed_CCN P3MainData(1, 10, 1, 72, 1, 1.800E+03, false, true), P3MainData(1, 10, 1, 72, 1, 1.800E+03, true, false), }; - static constexpr Int num_runs = sizeof(isds_fortran) / sizeof(P3MainData); + static constexpr Int num_runs = sizeof(isds_baseline) / sizeof(P3MainData); - for (auto& d : isds_fortran) { + for (auto& d : isds_baseline) { d.randomize(engine, { {d.pres , {1.00000000E+02 , 9.87111111E+04}}, {d.dz , {1.22776609E+02 , 3.49039167E+04}}, @@ -409,36 +429,36 @@ static void run_bfb_p3_main() }); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state P3MainData isds_cxx[num_runs] = { - P3MainData(isds_fortran[0]), - P3MainData(isds_fortran[1]), + P3MainData(isds_baseline[0]), + P3MainData(isds_baseline[1]), }; - // Get data from fortran - for (auto& d : isds_fortran) { - p3_main(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : isds_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : isds_cxx) { - d.template transpose(); - p3_main_f( + p3_main_host( d.qc, d.nc, d.qr, d.nr, d.th_atm, d.qv, d.dt, d.qi, d.qm, d.ni, d.bm, d.pres, d.dz, d.nc_nuceat_tend, d.nccn_prescribed, d.ni_activated, d.inv_qc_relvar, d.it, d.precip_liq_surf, d.precip_ice_surf, d.its, d.ite, d.kts, d.kte, d.diag_eff_radius_qc, d.diag_eff_radius_qi, d.diag_eff_radius_qr, d.rho_qi, d.do_predict_nc, d.do_prescribed_CCN, d.dpres, d.inv_exner, d.qv2qi_depos_tend, d.precip_liq_flux, d.precip_ice_flux, d.cld_frac_r, d.cld_frac_l, d.cld_frac_i, d.liq_ice_exchange, d.vap_liq_exchange, d.vap_ice_exchange, d.qv_prev, d.t_prev); - d.template transpose(); } - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - const auto& df90 = isds_fortran[i]; - const auto& dcxx = isds_fortran[i]; - const auto tot = isds_fortran[i].total(df90.qc); + const auto& df90 = isds_baseline[i]; + const auto& dcxx = isds_baseline[i]; + const auto tot = isds_baseline[i].total(df90.qc); for (Int t = 0; t < tot; ++t) { REQUIRE(df90.qc[t] == dcxx.qc[t]); REQUIRE(df90.nc[t] == dcxx.nc[t]); @@ -474,9 +494,14 @@ static void run_bfb_p3_main() REQUIRE(df90.precip_ice_surf[tot] == dcxx.precip_ice_surf[tot]); } } + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + isds_cxx[i].write(Base::m_fid); + } + } } -static void run_bfb() +void run_bfb() { run_bfb_p3_main_part1(); run_bfb_p3_main_part2(); @@ -494,12 +519,11 @@ namespace { TEST_CASE("p3_main", "[p3_functions]") { - using TP3 = scream::p3::unit_test::UnitWrap::UnitTest::TestP3Main; - - TP3::run_phys(); - TP3::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestP3Main; - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp index ffe7ea504e13..175247bb501d 100644 --- a/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp @@ -4,8 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" -#include "share/util/scream_setup_random_test.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -14,31 +13,33 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestNcConservation { +struct UnitWrap::UnitTest::TestNcConservation : public UnitWrap::UnitTest::Base { - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - NcConservationData f90_data[max_pack_size]; + NcConservationData baseline_data[max_pack_size]; // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { + // Alternatively, you can use the baseline_data construtors/initializer lists to hardcode data + for (auto& d : baseline_data) { d.randomize(engine); - d.dt = f90_data[0].dt; // Hold this fixed, this is not packed data + d.dt = baseline_data[0].dt; // Hold this fixed, this is not packed data } - // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that + // Create copies of data for use by cxx and sync it to device. Needs to happen before reads so that // inout data is in original state view_1d cxx_device("cxx_device", max_pack_size); const auto cxx_host = Kokkos::create_mirror_view(cxx_device); - std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); + std::copy(&baseline_data[0], &baseline_data[0] + max_pack_size, cxx_host.data()); Kokkos::deep_copy(cxx_device, cxx_host); - // Get data from fortran - for (auto& d : f90_data) { - nc_conservation(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + baseline_data[i].read(Base::m_fid); + } } // Get data from cxx. Run nc_conservation from a kernel and copy results back to host @@ -71,14 +72,19 @@ struct UnitWrap::UnitTest::TestNcConservation { Kokkos::deep_copy(cxx_host, cxx_device); // Verify BFB results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - NcConservationData& d_f90 = f90_data[i]; + NcConservationData& d_baseline = baseline_data[i]; NcConservationData& d_cxx = cxx_host[i]; - REQUIRE(d_f90.nc_collect_tend == d_cxx.nc_collect_tend); - REQUIRE(d_f90.nc2ni_immers_freeze_tend == d_cxx.nc2ni_immers_freeze_tend); - REQUIRE(d_f90.nc_accret_tend == d_cxx.nc_accret_tend); - REQUIRE(d_f90.nc2nr_autoconv_tend == d_cxx.nc2nr_autoconv_tend); + REQUIRE(d_baseline.nc_collect_tend == d_cxx.nc_collect_tend); + REQUIRE(d_baseline.nc2ni_immers_freeze_tend == d_cxx.nc2ni_immers_freeze_tend); + REQUIRE(d_baseline.nc_accret_tend == d_cxx.nc_accret_tend); + REQUIRE(d_baseline.nc2nr_autoconv_tend == d_cxx.nc2nr_autoconv_tend); + } + } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + cxx_host(s).write(Base::m_fid); } } } // run_bfb @@ -93,9 +99,10 @@ namespace { TEST_CASE("nc_conservation_bfb", "[p3]") { - using TestStruct = scream::p3::unit_test::UnitWrap::UnitTest::TestNcConservation; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestNcConservation; - TestStruct::run_bfb(); + T t; + t.run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp index 3aa3428825cd..b8cfdf4333a1 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp @@ -4,8 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" -#include "share/util/scream_setup_random_test.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -14,31 +13,33 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestNiConservation { +struct UnitWrap::UnitTest::TestNiConservation : public UnitWrap::UnitTest::Base { - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - NiConservationData f90_data[max_pack_size]; + NiConservationData baseline_data[max_pack_size]; // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { + // Alternatively, you can use the baseline_data construtors/initializer lists to hardcode data + for (auto& d : baseline_data) { d.randomize(engine); - d.dt = f90_data[0].dt; // hold dt fixed, it is not packed data + d.dt = baseline_data[0].dt; // hold dt fixed, it is not packed data } - // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that + // Create copies of data for use by cxx and sync it to device. Needs to happen before reads so that // inout data is in original state view_1d cxx_device("cxx_device", max_pack_size); const auto cxx_host = Kokkos::create_mirror_view(cxx_device); - std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); + std::copy(&baseline_data[0], &baseline_data[0] + max_pack_size, cxx_host.data()); Kokkos::deep_copy(cxx_device, cxx_host); - // Get data from fortran - for (auto& d : f90_data) { - ni_conservation(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + baseline_data[i].read(Base::m_fid); + } } // Get data from cxx. Run ni_conservation from a kernel and copy results back to host @@ -71,17 +72,21 @@ struct UnitWrap::UnitTest::TestNiConservation { Kokkos::deep_copy(cxx_host, cxx_device); // Verify BFB results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - NiConservationData& d_f90 = f90_data[i]; + NiConservationData& d_baseline = baseline_data[i]; NiConservationData& d_cxx = cxx_host[i]; - REQUIRE(d_f90.ni2nr_melt_tend == d_cxx.ni2nr_melt_tend); - REQUIRE(d_f90.ni_sublim_tend == d_cxx.ni_sublim_tend); - REQUIRE(d_f90.ni_selfcollect_tend == d_cxx.ni_selfcollect_tend); + REQUIRE(d_baseline.ni2nr_melt_tend == d_cxx.ni2nr_melt_tend); + REQUIRE(d_baseline.ni_sublim_tend == d_cxx.ni_sublim_tend); + REQUIRE(d_baseline.ni_selfcollect_tend == d_cxx.ni_selfcollect_tend); + } + } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + cxx_host(s).write(Base::m_fid); } } } // run_bfb - }; } // namespace unit_test @@ -92,9 +97,10 @@ namespace { TEST_CASE("ni_conservation_bfb", "[p3]") { - using TestStruct = scream::p3::unit_test::UnitWrap::UnitTest::TestNiConservation; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestNiConservation; - TestStruct::run_bfb(); + T t; + t.run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp index fb9e1e758b42..dc9a44af88d0 100644 --- a/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp @@ -4,8 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" -#include "share/util/scream_setup_random_test.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -14,32 +13,34 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestNrConservation { +struct UnitWrap::UnitTest::TestNrConservation : public UnitWrap::UnitTest::Base { - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - NrConservationData f90_data[max_pack_size]; + NrConservationData baseline_data[max_pack_size]; // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { + // Alternatively, you can use the baseline_data construtors/initializer lists to hardcode data + for (auto& d : baseline_data) { d.randomize(engine); - d.dt = f90_data[0].dt; // hold dt fixed, it is not packed data - d.nmltratio = f90_data[0].nmltratio; // hold nmltratio fixed, it is not packed data + d.dt = baseline_data[0].dt; // hold dt fixed, it is not packed data + d.nmltratio = baseline_data[0].nmltratio; // hold nmltratio fixed, it is not packed data } - // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that + // Create copies of data for use by cxx and sync it to device. Needs to happen before reads so that // inout data is in original state view_1d cxx_device("cxx_device", max_pack_size); const auto cxx_host = Kokkos::create_mirror_view(cxx_device); - std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); + std::copy(&baseline_data[0], &baseline_data[0] + max_pack_size, cxx_host.data()); Kokkos::deep_copy(cxx_device, cxx_host); - // Get data from fortran - for (auto& d : f90_data) { - nr_conservation(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + baseline_data[i].read(Base::m_fid); + } } // Get data from cxx. Run nr_conservation from a kernel and copy results back to host @@ -75,14 +76,19 @@ struct UnitWrap::UnitTest::TestNrConservation { Kokkos::deep_copy(cxx_host, cxx_device); // Verify BFB results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - NrConservationData& d_f90 = f90_data[i]; + NrConservationData& d_baseline = baseline_data[i]; NrConservationData& d_cxx = cxx_host[i]; - REQUIRE(d_f90.nr_collect_tend == d_cxx.nr_collect_tend); - REQUIRE(d_f90.nr2ni_immers_freeze_tend == d_cxx.nr2ni_immers_freeze_tend); - REQUIRE(d_f90.nr_selfcollect_tend == d_cxx.nr_selfcollect_tend); - REQUIRE(d_f90.nr_evap_tend == d_cxx.nr_evap_tend); + REQUIRE(d_baseline.nr_collect_tend == d_cxx.nr_collect_tend); + REQUIRE(d_baseline.nr2ni_immers_freeze_tend == d_cxx.nr2ni_immers_freeze_tend); + REQUIRE(d_baseline.nr_selfcollect_tend == d_cxx.nr_selfcollect_tend); + REQUIRE(d_baseline.nr_evap_tend == d_cxx.nr_evap_tend); + } + } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + cxx_host(s).write(Base::m_fid); } } } // run_bfb @@ -97,9 +103,10 @@ namespace { TEST_CASE("nr_conservation_bfb", "[p3]") { - using TestStruct = scream::p3::unit_test::UnitWrap::UnitTest::TestNrConservation; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestNrConservation; - TestStruct::run_bfb(); + T t; + t.run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp index a1b270562819..1b439aca7aac 100644 --- a/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp @@ -3,8 +3,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" -#include "share/util/scream_setup_random_test.hpp" +#include "p3_test_data.hpp" #include "share/scream_types.hpp" #include "physics/share/physics_functions.hpp" @@ -15,9 +14,9 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { +struct UnitWrap::UnitTest::TestPreventLiqSupersaturation : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() //Conceptual tests for prevent_liq_supersaturation. Note many conceptual tests make sense to run on //random data, so are included in run_bfb rather than here. { @@ -88,32 +87,32 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { } //end run_property - static void run_bfb() + void run_bfb() { constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - PreventLiqSupersaturationData f90_data[max_pack_size]; + PreventLiqSupersaturationData baseline_data[max_pack_size]; // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { + // Alternatively, you can use the baseline_data construtors/initializer lists to hardcode data + for (auto& d : baseline_data) { d.randomize(engine); - d.dt = f90_data[0].dt; // Hold this fixed, this is not packed data + d.dt = baseline_data[0].dt; // Hold this fixed, this is not packed data // C++ impl uses constants for latent_heat values. Manually set here - // so F90 can match + // so BASELINE can match d.latent_heat_vapor = latvap; d.latent_heat_sublim = latvap+latice; } // Create copies of data for use by cxx and sync it to device. Needs to happen before - // fortran calls so that inout data is in original state + // reads so that inout data is in original state view_1d cxx_device("cxx_device", max_pack_size); const auto cxx_host = Kokkos::create_mirror_view(cxx_device); - std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); + std::copy(&baseline_data[0], &baseline_data[0] + max_pack_size, cxx_host.data()); Kokkos::deep_copy(cxx_device, cxx_host); // Save copy of inout vars to check that prevent_liq_supersaturation always makes them smaller @@ -123,9 +122,11 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { qr2qv_evap_tend_init[i] = cxx_host(i).qr2qv_evap_tend; } - // Get data from fortran - for (auto& d : f90_data) { - prevent_liq_supersaturation(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + baseline_data[i].read(Base::m_fid); + } } // Get data from cxx. Run prevent_liq_supersaturation from a kernel and copy results back to host @@ -158,13 +159,13 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { Kokkos::deep_copy(cxx_host, cxx_device); - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { // Verify BFB results - PreventLiqSupersaturationData& d_f90 = f90_data[i]; + PreventLiqSupersaturationData& d_baseline = baseline_data[i]; PreventLiqSupersaturationData& d_cxx = cxx_host[i]; - REQUIRE(d_f90.qi2qv_sublim_tend == d_cxx.qi2qv_sublim_tend); - REQUIRE(d_f90.qr2qv_evap_tend == d_cxx.qr2qv_evap_tend); + REQUIRE(d_baseline.qi2qv_sublim_tend == d_cxx.qi2qv_sublim_tend); + REQUIRE(d_baseline.qr2qv_evap_tend == d_cxx.qr2qv_evap_tend); //Verify tendencies are always >=0: REQUIRE(d_cxx.qi2qv_sublim_tend>=0); @@ -175,8 +176,12 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { REQUIRE(d_cxx.qr2qv_evap_tend<=qr2qv_evap_tend_init[i]); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + cxx_host(s).write(Base::m_fid); + } + } } // run_bfb - }; } // namespace unit_test @@ -187,14 +192,18 @@ namespace { TEST_CASE("prevent_liq_supersaturation_property", "[p3]") { - using TestStruct = scream::p3::unit_test::UnitWrap::UnitTest::TestPreventLiqSupersaturation; - TestStruct::run_property(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestPreventLiqSupersaturation; + + T t; + t.run_property(); } TEST_CASE("prevent_liq_supersaturation_bfb", "[p3]") { - using TestStruct = scream::p3::unit_test::UnitWrap::UnitTest::TestPreventLiqSupersaturation; - TestStruct::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestPreventLiqSupersaturation; + + T t; + t.run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp index b74f734e87c4..935842cb6d98 100644 --- a/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -18,14 +18,14 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestRainImmersionFreezing { +struct UnitWrap::UnitTest::TestRainImmersionFreezing : public UnitWrap::UnitTest::Base { -static void run_phys() +void run_phys() { // TODO } -static void run_bfb() +void run_bfb() { // This is the threshold for whether the qc and qr cloud mixing ratios are // large enough to affect the warm-phase process rates qc2qr_accret_tend and nc_accret_tend. @@ -69,9 +69,11 @@ static void run_bfb() host_data.data()); Kokkos::deep_copy(device_data, host_data); - // Run the Fortran subroutine. - for (Int i = 0; i < max_pack_size; ++i) { - rain_immersion_freezing(rain_imm_freezing_data[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + rain_imm_freezing_data[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -91,8 +93,9 @@ static void run_bfb() Spack qr2qi_immers_freeze_tend{0.0}; Spack nr2ni_immers_freeze_tend{0.0}; - Functions::rain_immersion_freezing(T_atm, lamr, mu_r, cdistr, qr_incld, - qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend, physics::P3_Constants()); + Functions::rain_immersion_freezing( + T_atm, lamr, mu_r, cdistr, qr_incld, qr2qi_immers_freeze_tend, + nr2ni_immers_freeze_tend, p3::Functions::P3Runtime()); // Copy results back into views for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { @@ -105,12 +108,17 @@ static void run_bfb() Kokkos::deep_copy(host_data, device_data); // Validate results. - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(rain_imm_freezing_data[s].qr2qi_immers_freeze_tend == host_data[s].qr2qi_immers_freeze_tend); REQUIRE(rain_imm_freezing_data[s].nr2ni_immers_freeze_tend == host_data[s].nr2ni_immers_freeze_tend); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + host_data(s).write(Base::m_fid); + } + } } }; @@ -123,12 +131,11 @@ namespace { TEST_CASE("p3_rain_immersion_freezing", "[p3_functions]") { - using TRIF = scream::p3::unit_test::UnitWrap::UnitTest::TestRainImmersionFreezing; - - TRIF::run_phys(); - TRIF::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestRainImmersionFreezing; - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp index dfbf5f3a2f1f..ac34856eea62 100644 --- a/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp @@ -4,9 +4,8 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" -#include "p3_f90.hpp" -#include "share/util/scream_setup_random_test.hpp" +#include "p3_test_data.hpp" +#include "p3_data.hpp" #include "p3_unit_tests_common.hpp" @@ -20,33 +19,33 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestRainSed { +struct UnitWrap::UnitTest::TestRainSed : public UnitWrap::UnitTest::Base { -static void run_phys_rain_vel() +void run_phys_rain_vel() { // TODO } -static void run_phys_rain_sed() +void run_phys_rain_sed() { // TODO } -static void run_phys() +void run_phys() { run_phys_rain_vel(); run_phys_rain_sed(); } -static void run_bfb_rain_vel() +void run_bfb_rain_vel() { // Read in tables view_2d_table vn_table_vals; view_2d_table vm_table_vals; view_2d_table revap_table_vals; view_1d_table mu_r_table_vals; view_dnu_table dnu; - Functions::init_kokkos_tables(vn_table_vals, vm_table_vals, revap_table_vals, mu_r_table_vals, dnu); + Functions::get_global_tables(vn_table_vals, vm_table_vals, revap_table_vals, mu_r_table_vals, dnu); // Load some lookup inputs, need at least one per pack value - ComputeRainFallVelocityData crfv_fortran[max_pack_size] = { + ComputeRainFallVelocityData crfv_baseline[max_pack_size] = { // qr_incld, rhofacr, nr_incld {1.1030E-04, 1.3221E+00, 6.2964E+05}, {2.1437E-13, 1.0918E+00, 6.5337E+07}, @@ -70,16 +69,18 @@ static void run_bfb_rain_vel() }; - // Sync to device, needs to happen before fortran calls so that + // Sync to device, needs to happen before reads so that // inout data is in original state view_1d crfv_device("crfv", max_pack_size); const auto crfv_host = Kokkos::create_mirror_view(crfv_device); - std::copy(&crfv_fortran[0], &crfv_fortran[0] + max_pack_size, crfv_host.data()); + std::copy(&crfv_baseline[0], &crfv_baseline[0] + max_pack_size, crfv_host.data()); Kokkos::deep_copy(crfv_device, crfv_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - compute_rain_fall_velocity(crfv_fortran[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + crfv_baseline[i].read(Base::m_fid); + } } // Calc bulk rime from a kernel and copy results back to host @@ -96,7 +97,8 @@ static void run_bfb_rain_vel() Spack mu_r(0), lamr(0), V_qr(0), V_nr(0); Functions::compute_rain_fall_velocity( - vn_table_vals, vm_table_vals, qr_incld, rhofacr, nr_incld, mu_r, lamr, V_qr, V_nr, physics::P3_Constants()); + vn_table_vals, vm_table_vals, qr_incld, rhofacr, nr_incld, mu_r, lamr, + V_qr, V_nr, p3::Functions::P3Runtime()); // Copy results back into views for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { @@ -111,21 +113,27 @@ static void run_bfb_rain_vel() // Sync back to host Kokkos::deep_copy(crfv_host, crfv_device); - // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + // Validate results for (Int s = 0; s < max_pack_size; ++s) { - REQUIRE(crfv_fortran[s].nr_incld == crfv_host(s).nr_incld); - REQUIRE(crfv_fortran[s].mu_r == crfv_host(s).mu_r); - REQUIRE(crfv_fortran[s].lamr == crfv_host(s).lamr); - REQUIRE(crfv_fortran[s].V_qr == crfv_host(s).V_qr); - REQUIRE(crfv_fortran[s].V_nr == crfv_host(s).V_nr); + REQUIRE(crfv_baseline[s].nr_incld == crfv_host(s).nr_incld); + REQUIRE(crfv_baseline[s].mu_r == crfv_host(s).mu_r); + REQUIRE(crfv_baseline[s].lamr == crfv_host(s).lamr); + REQUIRE(crfv_baseline[s].V_qr == crfv_host(s).V_qr); + REQUIRE(crfv_baseline[s].V_nr == crfv_host(s).V_nr); + } + } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + crfv_host(s).write(Base::m_fid); } } } -static void run_bfb_rain_sed() +void run_bfb_rain_sed() { - auto engine = setup_random_test(); + // With stored baselines, we must use a fixed seed! + auto engine = Base::get_engine(); // F90 is quite slow on weaver, so we decrease dt to reduce // the number of steps in rain_sed. @@ -135,7 +143,7 @@ static void run_bfb_rain_sed() constexpr Scalar dt = 1.800E+03; #endif - RainSedData rsds_fortran[] = { + RainSedData rsds_baseline[] = { // kts, kte, ktop, kbot, kdir, dt, inv_dt, precip_liq_surf RainSedData(1, 72, 27, 72, -1, dt, 1/dt, 0.0), RainSedData(1, 72, 72, 27, 1, dt, 1/dt, 1.0), @@ -143,25 +151,27 @@ static void run_bfb_rain_sed() RainSedData(1, 72, 27, 27, 1, dt, 1/dt, 2.0), }; - static constexpr Int num_runs = sizeof(rsds_fortran) / sizeof(RainSedData); + static constexpr Int num_runs = sizeof(rsds_baseline) / sizeof(RainSedData); // Set up random input data - for (auto& d : rsds_fortran) { + for (auto& d : rsds_baseline) { d.randomize(engine, { {d.qr_incld, {C::QSMALL/2, C::QSMALL*2}} }); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state RainSedData rsds_cxx[num_runs] = { - RainSedData(rsds_fortran[0]), - RainSedData(rsds_fortran[1]), - RainSedData(rsds_fortran[2]), - RainSedData(rsds_fortran[3]), + RainSedData(rsds_baseline[0]), + RainSedData(rsds_baseline[1]), + RainSedData(rsds_baseline[2]), + RainSedData(rsds_baseline[3]), }; - // Get data from fortran - for (auto& d : rsds_fortran) { - rain_sedimentation(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : rsds_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx @@ -172,35 +182,40 @@ static void run_bfb_rain_sed() #if defined(SCREAM_FORCE_RUN_DIFF) inv_dt *= 2; #endif - rain_sedimentation_f(d.kts, d.kte, d.ktop, d.kbot, d.kdir, + rain_sedimentation_host(d.kts, d.kte, d.ktop, d.kbot, d.kdir, d.qr_incld, d.rho, d.inv_rho, d.rhofacr, d.cld_frac_r, d.inv_dz, d.dt, inv_dt, d.qr, d.nr, d.nr_incld, d.mu_r, d.lamr, &d.precip_liq_surf, d.precip_liq_flux, d.qr_tend, d.nr_tend); } - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { // Due to pack issues, we must restrict checks to the active k space - Int start = std::min(rsds_fortran[i].kbot, rsds_fortran[i].ktop) - 1; // 0-based indx - Int end = std::max(rsds_fortran[i].kbot, rsds_fortran[i].ktop); // 0-based indx + Int start = std::min(rsds_baseline[i].kbot, rsds_baseline[i].ktop) - 1; // 0-based indx + Int end = std::max(rsds_baseline[i].kbot, rsds_baseline[i].ktop); // 0-based indx for (Int k = start; k < end; ++k) { - REQUIRE(rsds_fortran[i].qr[k] == rsds_cxx[i].qr[k]); - REQUIRE(rsds_fortran[i].nr[k] == rsds_cxx[i].nr[k]); - REQUIRE(rsds_fortran[i].nr_incld[k] == rsds_cxx[i].nr_incld[k]); - REQUIRE(rsds_fortran[i].mu_r[k] == rsds_cxx[i].mu_r[k]); - REQUIRE(rsds_fortran[i].lamr[k] == rsds_cxx[i].lamr[k]); - REQUIRE(rsds_fortran[i].precip_liq_flux[k] == rsds_cxx[i].precip_liq_flux[k]); - REQUIRE(rsds_fortran[i].qr_tend[k] == rsds_cxx[i].qr_tend[k]); - REQUIRE(rsds_fortran[i].nr_tend[k] == rsds_cxx[i].nr_tend[k]); + REQUIRE(rsds_baseline[i].qr[k] == rsds_cxx[i].qr[k]); + REQUIRE(rsds_baseline[i].nr[k] == rsds_cxx[i].nr[k]); + REQUIRE(rsds_baseline[i].nr_incld[k] == rsds_cxx[i].nr_incld[k]); + REQUIRE(rsds_baseline[i].mu_r[k] == rsds_cxx[i].mu_r[k]); + REQUIRE(rsds_baseline[i].lamr[k] == rsds_cxx[i].lamr[k]); + REQUIRE(rsds_baseline[i].precip_liq_flux[k] == rsds_cxx[i].precip_liq_flux[k]); + REQUIRE(rsds_baseline[i].qr_tend[k] == rsds_cxx[i].qr_tend[k]); + REQUIRE(rsds_baseline[i].nr_tend[k] == rsds_cxx[i].nr_tend[k]); } - REQUIRE(rsds_fortran[i].precip_liq_flux[end] == rsds_cxx[i].precip_liq_flux[end]); - REQUIRE(rsds_fortran[i].precip_liq_surf == rsds_cxx[i].precip_liq_surf); + REQUIRE(rsds_baseline[i].precip_liq_flux[end] == rsds_cxx[i].precip_liq_flux[end]); + REQUIRE(rsds_baseline[i].precip_liq_surf == rsds_cxx[i].precip_liq_surf); + } + } + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + rsds_cxx[i].write(Base::m_fid); } } } -static void run_bfb() +void run_bfb() { run_bfb_rain_vel(); run_bfb_rain_sed(); @@ -216,14 +231,11 @@ namespace { TEST_CASE("p3_rain_sed", "[p3_functions]") { - using TRS = scream::p3::unit_test::UnitWrap::UnitTest::TestRainSed; - - scream::p3::p3_init(); // need fortran table data - - TRS::run_phys(); - TRS::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestRainSed; - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp index 5adc3ed71bab..26440b6680bf 100644 --- a/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -22,9 +22,9 @@ namespace unit_test { * Unit-tests for p3 ice collection functions. */ template -struct UnitWrap::UnitTest::TestRainSelfCollection { +struct UnitWrap::UnitTest::TestRainSelfCollection : public UnitWrap::UnitTest::Base { - static void run_rain_self_collection_bfb_tests(){ + void run_rain_self_collection_bfb_tests() { RainSelfCollectionData dc[max_pack_size] = { // rho, qr_incld, nr_incld, nr_selfcollect_tend @@ -56,9 +56,11 @@ struct UnitWrap::UnitTest::TestRainSelfCollection { std::copy(&dc[0], &dc[0] + max_pack_size, dc_host.data()); Kokkos::deep_copy(dc_device, dc_host); - //Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - rain_self_collection(dc[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + dc[i].read(Base::m_fid); + } } //Run function from a kernal and copy results back to the host @@ -74,8 +76,9 @@ struct UnitWrap::UnitTest::TestRainSelfCollection { nr_selfcollect_tend_local[s] = dc_device(vs).nr_selfcollect_tend; } - Functions::rain_self_collection(rho_local, qr_incld_local, nr_incld_local, nr_selfcollect_tend_local, - physics::P3_Constants()); + Functions::rain_self_collection( + rho_local, qr_incld_local, nr_incld_local, nr_selfcollect_tend_local, + p3::Functions::P3Runtime()); // Copy results back into views for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { @@ -90,7 +93,7 @@ struct UnitWrap::UnitTest::TestRainSelfCollection { Kokkos::deep_copy(dc_host, dc_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(dc[s].rho == dc_host(s).rho); REQUIRE(dc[s].qr_incld == dc_host(s).qr_incld); @@ -98,9 +101,14 @@ struct UnitWrap::UnitTest::TestRainSelfCollection { REQUIRE(dc[s].nr_selfcollect_tend == dc_host(s).nr_selfcollect_tend); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + dc_host(s).write(Base::m_fid); + } + } } - static void run_bfb(){ + void run_bfb() { run_rain_self_collection_bfb_tests(); } @@ -113,7 +121,10 @@ struct UnitWrap::UnitTest::TestRainSelfCollection { namespace { TEST_CASE("p3_rain_self_collection_test", "[p3_rain_self_collection_test"){ - scream::p3::unit_test::UnitWrap::UnitTest::TestRainSelfCollection::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestRainSelfCollection; + + T t; + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp b/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp index 5bc12b2464db..f596b0910eb2 100644 --- a/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp @@ -3,7 +3,7 @@ #include "share/util/scream_utils.hpp" #include "p3_main_wrap.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_ic_cases.hpp" #include "ekat/util/ekat_file_utils.hpp" @@ -16,6 +16,7 @@ namespace { using namespace scream; using namespace scream::p3; +using P3F = Functions; /* * p3_run_and_cmp can be run in 2 modes. First, generate_baseline @@ -35,10 +36,10 @@ using namespace scream::p3; * large discrepancies. */ Int compare (const double& tol, - const FortranData::Ptr& ref, const FortranData::Ptr& d) { + const P3Data::Ptr& ref, const P3Data::Ptr& d) { Int nerr = 0; - FortranDataIterator refi(ref), di(d); + P3DataIterator refi(ref), di(d); EKAT_ASSERT(refi.nfield() == di.nfield()); for (Int i = 0, n = refi.nfield(); i < n; ++i) { const auto& fr = refi.getfield(i); @@ -75,7 +76,7 @@ struct Baseline { } } - Int generate_baseline (const std::string& filename, bool use_fortran) { + Int generate_baseline (const std::string& filename) { auto fid = ekat::FILEPtr(fopen(filename.c_str(), "w")); EKAT_REQUIRE_MSG( fid, "generate_baseline can't write " << filename); Int nerr = 0; @@ -87,22 +88,19 @@ struct Baseline { for (Int r = -1; r < ps.repeat; ++r) { const auto d = ic::Factory::create(ps.ic, ps.ncol, ps.nlev); set_params(ps, *d); - p3_init(); + P3F::p3_init(); if (ps.repeat > 0 && r == -1) { std::cout << "Running P3 with ni=" << d->ncol << ", nk=" << d->nlev << ", dt=" << d->dt << ", ts=" << d->it << ", predict_nc=" << d->do_predict_nc - << ", prescribed_CCN=" << d->do_prescribed_CCN; - - if (!use_fortran) { - std::cout << ", small_packn=" << SCREAM_SMALL_PACK_SIZE; - } - std::cout << std::endl; + << ", prescribed_CCN=" << d->do_prescribed_CCN + << ", small_packn=" << SCREAM_SMALL_PACK_SIZE + << std::endl; } for (int it=0; it 0) { // do not count the "cold" run total_duration_microsec += current_microsec; @@ -123,29 +121,43 @@ struct Baseline { return nerr; } - Int run_and_cmp (const std::string& filename, const double& tol, bool use_fortran) { - auto fid = ekat::FILEPtr(fopen(filename.c_str(), "r")); - EKAT_REQUIRE_MSG( fid, "generate_baseline can't read " << filename); + Int run_and_cmp (const std::string& filename, const double& tol, bool no_baseline) { + ekat::FILEPtr fid; + if (!no_baseline) { + fid = ekat::FILEPtr(fopen(filename.c_str(), "r")); + EKAT_REQUIRE_MSG( fid, "generate_baseline can't read " << filename); + } Int nerr = 0, ne; int case_num = 0; for (auto ps : params_) { case_num++; - // Read the reference impl's data from the baseline file. - const auto d_ref = ic::Factory::create(ps.ic, ps.ncol, ps.nlev); - set_params(ps, *d_ref); - // Now run a sequence of other impls. This includes the reference - // implementation b/c it's likely we'll want to change it as we go. - { + if (no_baseline) { const auto d = ic::Factory::create(ps.ic, ps.ncol, ps.nlev); set_params(ps, *d); - p3_init(); + P3F::p3_init(); for (int it=0; it params_; - static void write (const ekat::FILEPtr& fid, const FortranData::Ptr& d) { - FortranDataIterator fdi(d); + static void write (const ekat::FILEPtr& fid, const P3Data::Ptr& d) { + P3DataIterator fdi(d); for (Int i = 0, n = fdi.nfield(); i < n; ++i) { const auto& f = fdi.getfield(i); ekat::write(&f.dim, 1, fid); @@ -182,8 +194,8 @@ struct Baseline { } } - static void read (const ekat::FILEPtr& fid, const FortranData::Ptr& d) { - FortranDataIterator fdi(d); + static void read (const ekat::FILEPtr& fid, const P3Data::Ptr& d) { + P3DataIterator fdi(d); for (Int i = 0, n = fdi.nfield(); i < n; ++i) { const auto& f = fdi.getfield(i); int dim, ds[3]; @@ -220,22 +232,24 @@ int main (int argc, char** argv) { if (argc == 1) { std::cout << - argv[0] << " [options] baseline-filename\n" + argv[0] << " [options] \n" "Options:\n" " -g Generate baseline file. Default False.\n" - " -f Use fortran impls instead of c++. Default False.\n" + " -c Compare baseline file. Default False.\n" + " -n Run without baseline actions. Default True.\n" + " -b Path to directory containing baselines.\n" " -t Tolerance for relative error. Default 0.\n" " -s Number of timesteps. Default=6.\n" " -dt Length of timestep. Default=300.\n" " -i Number of columns. Default=3.\n" " -k Number of vertical levels. Default=72.\n" " -r Number of repetitions, implies timing run (generate + no I/O). Default=0.\n" - " -p yes|no|both. Default=both.\n" - " -c yes|no|both. Default=both.\n"; + " --predict-nc yes|no|both. Default=both.\n" + " --prescribed-ccn yes|no|both. Default=both.\n"; return 1; } - bool generate = false, use_fortran = false; + bool generate = false, no_baseline = true; scream::Real tol = SCREAM_BFB_TESTING ? 0 : std::numeric_limits::infinity(); Int timesteps = 6; Int dt = 300; @@ -246,18 +260,18 @@ int main (int argc, char** argv) { std::string predict_nc = "both"; std::string prescribed_ccn = "both"; std::string baseline_fn; - for (int i = 1; i < argc-1; ++i) { - if (ekat::argv_matches(argv[i], "-g", "--generate")) generate = true; - if (ekat::argv_matches(argv[i], "-f", "--fortran")) use_fortran = true; - if (ekat::argv_matches(argv[i], "-t", "--tol")) { + for (int i = 1; i < argc; ++i) { + if (ekat::argv_matches(argv[i], "-g", "--generate")) { generate = true; no_baseline = false; } + if (ekat::argv_matches(argv[i], "-c", "--compare")) { no_baseline = false; } + if (ekat::argv_matches(argv[i], "-b", "--baseline-file")) { expect_another_arg(i, argc); ++i; - tol = std::atof(argv[i]); + baseline_fn = argv[i]; } - if (ekat::argv_matches(argv[i], "-b", "--baseline-file")) { + if (ekat::argv_matches(argv[i], "-t", "--tol")) { expect_another_arg(i, argc); ++i; - baseline_fn = argv[i]; + tol = std::atof(argv[i]); } if (ekat::argv_matches(argv[i], "-s", "--steps")) { expect_another_arg(i, argc); @@ -279,27 +293,23 @@ int main (int argc, char** argv) { ++i; nlev = std::atoi(argv[i]); } - if (std::string(argv[i])=="--ekat-kokkos-device") { - expect_another_arg(i, argc); - ++i; - device = argv[i]; + if (std::string(argv[i])=="--kokkos-device-id=") { + auto tokens = ekat::split(argv[i],"="); + device = tokens[1]; } if (ekat::argv_matches(argv[i], "-r", "--repeat")) { expect_another_arg(i, argc); ++i; repeat = std::atoi(argv[i]); - if (repeat > 0) { - generate = true; - } } - if (ekat::argv_matches(argv[i], "-p", "--predict-nc")) { + if (ekat::argv_matches(argv[i], "-pn", "--predict-nc")) { expect_another_arg(i, argc); ++i; predict_nc = std::string(argv[i]); EKAT_REQUIRE_MSG(predict_nc == "yes" || predict_nc == "no" || predict_nc == "both", "Predict option value must be one of yes|no|both"); } - if (ekat::argv_matches(argv[i], "-c", "--prescribed-ccn")) { + if (ekat::argv_matches(argv[i], "-pc", "--prescribed-ccn")) { expect_another_arg(i, argc); ++i; prescribed_ccn = std::string(argv[i]); @@ -308,45 +318,25 @@ int main (int argc, char** argv) { } } - // Decorate baseline name with precision. - baseline_fn += std::to_string(sizeof(scream::Real)); - - std::vector args; - for (int i=0; i" was specified, add kokkos - // initialization flag to argv - // Create it outside the if, so its c_str pointer survives - std::string dev_arg; - if (device!="") { - auto is_int = [] (const std::string& s)->bool { - std::istringstream is(s); - int d; - is >> d; - return !is.fail() && is.eof(); - }; - - EKAT_REQUIRE_MSG (is_int(device), "Error! Invalid device specification.\n"); - - if (std::stoi(device) != -1) { - dev_arg = "--kokkos-device-id=" + device; - args.push_back(const_cast(dev_arg.c_str())); - } - } + // Compute full baseline file name with precision. + baseline_fn += "/p3_run_and_cmp.baseline" + std::to_string(sizeof(scream::Real)); - scream::initialize_scream_session(args.size(), args.data()); { + scream::initialize_scream_session(argc, argv); + { Baseline bln(timesteps, static_cast(dt), ncol, nlev, repeat, predict_nc, prescribed_ccn); if (generate) { std::cout << "Generating to " << baseline_fn << "\n"; - nerr += bln.generate_baseline(baseline_fn, use_fortran); - } else { + nerr += bln.generate_baseline(baseline_fn); + } else if (no_baseline) { + printf("Running with no baseline actions\n"); + nerr += bln.run_and_cmp(baseline_fn, tol, no_baseline); + } + else { printf("Comparing with %s at tol %1.1e\n", baseline_fn.c_str(), tol); - nerr += bln.run_and_cmp(baseline_fn, tol, use_fortran); + nerr += bln.run_and_cmp(baseline_fn, tol, no_baseline); } - P3GlobalForFortran::deinit(); - } scream::finalize_scream_session(); + } + scream::finalize_scream_session(); return nerr != 0 ? 1 : 0; } diff --git a/components/eamxx/src/physics/p3/tests/p3_subgrid_variance_scaling_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_subgrid_variance_scaling_unit_tests.cpp index 5e0cf401bec8..809c98558f51 100644 --- a/components/eamxx/src/physics/p3/tests/p3_subgrid_variance_scaling_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_subgrid_variance_scaling_unit_tests.cpp @@ -3,7 +3,7 @@ #include "share/scream_types.hpp" #include "ekat/ekat_pack.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_unit_tests_common.hpp" @@ -19,11 +19,11 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling +struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling : public UnitWrap::UnitTest::Base { //----------------------------------------------------------------- - static void run_bfb_tests(){ + void run_bfb_tests() { //test that C++ and F90 implementations are BFB //Set of relvar values to loop over @@ -36,9 +36,7 @@ struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling //Set of exponents to loop over Scalar expons[3] = {1.0,2.47,0.1}; - //initialize struct required for F90 call - SubgridVarianceScalingData f_data; - Scalar f_scaling; + Scalar baseline_scaling; //Make C++ output available on host and device view_1d scaling_device("c scaling",1); @@ -47,11 +45,11 @@ struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling for (Int i = 0; i < 3; ++i) { // loop over exponents for (Int j = 0; j < 16; ++j) { // loop over relvars - // Get F90 solution + // Get baseline solution // ---------------------------------- - f_data.relvar=relvars[j]; - f_data.expon =expons[i]; - f_scaling = subgrid_variance_scaling(f_data); + if (this->m_baseline_action == COMPARE) { + ekat::read(&baseline_scaling, 1, Base::m_fid); + } // Get C++ solution // ---------------------------------- @@ -72,8 +70,11 @@ struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling Kokkos::deep_copy(scaling_host, scaling_device); // Validate results - if (SCREAM_BFB_TESTING) { - REQUIRE(f_scaling == scaling_host(0) ); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + REQUIRE(baseline_scaling == scaling_host(0) ); + } + else if (this->m_baseline_action == GENERATE) { + ekat::write(&scaling_host(0), 1, Base::m_fid); } } //end loop over relvar[j] } //end loop over expons[i] @@ -81,7 +82,7 @@ struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling //----------------------------------------------------------------- KOKKOS_FUNCTION static void subgrid_variance_scaling_linearity_test(const Scalar& relvar, - int& errors){ + int& errors) { //If expon=1, subgrid_variance_scaling should be 1 Scalar tol = C::macheps * 1e3; //1e3 is scale factor to make pass, essentially an estimate of numerical error @@ -97,7 +98,7 @@ struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling } //----------------------------------------------------------------- - KOKKOS_FUNCTION static void subgrid_variance_scaling_relvar1_test(int& errors){ + KOKKOS_FUNCTION static void subgrid_variance_scaling_relvar1_test(int& errors) { //If relvar=1, subgrid_variance_scaling should be factorial(expon) Scalar tol = C::macheps * 1e3; //1e3 is scale factor to make pass, essentially an estimate of numerical error @@ -116,7 +117,7 @@ struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling } //----------------------------------------------------------------- - KOKKOS_FUNCTION static void subgrid_variance_scaling_relvar3_test(int& errors){ + KOKKOS_FUNCTION static void subgrid_variance_scaling_relvar3_test(int& errors) { //If expon=3, subgrid variance scaling should be relvar^3+3*relvar^2+2*relvar/relvar^3 Scalar tol = C::macheps * 100; //100 is a fudge factor to make sure tests pass. 10 was too small for gnu on CPU. @@ -151,7 +152,7 @@ struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling } //end relvar3_test //----------------------------------------------------------------- - static void run_property_tests(){ + void run_property_tests() { /*This function executes all the SGS variance scaling tests by looping *over a bunch of test and summing their return statuses. *If that sum is zero, no errors have occurred. Otherwise you have errors. @@ -189,12 +190,14 @@ struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling } // namespace p3 } // namespace scream -namespace{ +namespace { TEST_CASE("p3_subgrid_variance_scaling_test", "[p3_subgrid_variance_scaling_test]"){ - scream::p3::unit_test::UnitWrap::UnitTest::TestP3SubgridVarianceScaling::run_bfb_tests(); - scream::p3::unit_test::UnitWrap::UnitTest::TestP3SubgridVarianceScaling::run_property_tests(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestP3SubgridVarianceScaling; + + T t; + t.run_bfb_tests(); + t.run_property_tests(); } } // namespace - diff --git a/components/eamxx/src/physics/p3/tests/p3_table3_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_table3_unit_tests.cpp index c99d156004e7..001a875c7e8c 100644 --- a/components/eamxx/src/physics/p3/tests/p3_table3_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_table3_unit_tests.cpp @@ -3,8 +3,8 @@ #include "p3_unit_tests_common.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" -#include "p3_f90.hpp" +#include "p3_test_data.hpp" +#include "p3_data.hpp" #include "share/scream_types.hpp" #include "ekat/ekat_pack.hpp" @@ -44,7 +44,7 @@ namespace unit_test { // refinement, where the mesh is a 1D mesh transecting the table domain. template -struct UnitWrap::UnitTest::TestTable3 { +struct UnitWrap::UnitTest::TestTable3 : public UnitWrap::UnitTest::Base { KOKKOS_FUNCTION static Scalar calc_lamr (const Scalar& mu_r, const Scalar& alpha) { // Parameters for lower and upper bounds, derived above, multiplied by @@ -68,7 +68,7 @@ struct UnitWrap::UnitTest::TestTable3 { return Functions::apply_table(table, t3); } - static void run () { + void run () { // This test doesn't use mu_r_table_vals, as that is not a table3 type. It // doesn't matter whether we use vm_table_vals or vn_table_vals, as the table values // don't matter in what we are testing; we are testing interpolation @@ -76,7 +76,7 @@ struct UnitWrap::UnitTest::TestTable3 { view_1d_table mu_r_table_vals; view_2d_table vn_table_vals, vm_table_vals, revap_table_vals; view_dnu_table dnu; - Functions::init_kokkos_tables(vn_table_vals, vm_table_vals, revap_table_vals, mu_r_table_vals, dnu); + Functions::get_global_tables(vn_table_vals, vm_table_vals, revap_table_vals, mu_r_table_vals, dnu); // Estimate two maximum slope magnitudes for two meshes, the second 10x // refined w.r.t. the first. @@ -170,9 +170,10 @@ namespace { TEST_CASE("p3_tables", "[p3_functions]") { - scream::p3::p3_init(); // need fortran table data + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestTable3; - scream::p3::unit_test::UnitWrap::UnitTest::TestTable3::run(); + T t; + t.run(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_tables_setup.cpp b/components/eamxx/src/physics/p3/tests/p3_tables_setup.cpp new file mode 100644 index 000000000000..ad9b299a4d9e --- /dev/null +++ b/components/eamxx/src/physics/p3/tests/p3_tables_setup.cpp @@ -0,0 +1,14 @@ +// This is a tiny program that calls p3_init() to generate tables used by p3 + +#include "physics/p3/p3_functions.hpp" +#include "share/scream_session.hpp" + +int main(int argc, char** argv) { + using P3F = scream::p3::Functions; + + scream::initialize_scream_session(argc, argv); + P3F::p3_init(/* write_tables = */ true, /* masterproc */ true); + scream::finalize_scream_session(); + + return 0; +} diff --git a/components/eamxx/src/physics/p3/tests/p3_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_tests.cpp index d1a64f48520c..858a231094d4 100644 --- a/components/eamxx/src/physics/p3/tests/p3_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_tests.cpp @@ -1,19 +1,19 @@ #include "catch2/catch.hpp" -#include "p3_f90.hpp" +#include "p3_data.hpp" #include "p3_main_wrap.hpp" #include "p3_ic_cases.hpp" namespace { -TEST_CASE("FortranData", "p3") { - int val = scream::p3::test_FortranData(); +TEST_CASE("P3Data", "p3") { + int val = scream::p3::test_P3Data(); REQUIRE(val == 0); } -TEST_CASE("FortranDataIterator", "p3") { +TEST_CASE("P3DataIterator", "p3") { using scream::p3::ic::Factory; const auto d = Factory::create(Factory::mixed); - scream::p3::FortranDataIterator fdi(d); + scream::p3::P3DataIterator fdi(d); REQUIRE(fdi.nfield() == 35); const auto& f = fdi.getfield(0); REQUIRE(f.dim == 2); @@ -29,13 +29,8 @@ TEST_CASE("p3_init", "p3") { REQUIRE(nerr == 0); } -TEST_CASE("p3_ic_f", "p3") { - int nerr = scream::p3::test_p3_ic(true); - REQUIRE(nerr == 0); -} - TEST_CASE("p3_ic_c", "p3") { - int nerr = scream::p3::test_p3_ic(false); + int nerr = scream::p3::test_p3_ic(); REQUIRE(nerr == 0); } diff --git a/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp index 3b994cfdad5d..91e3db2f2976 100644 --- a/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp @@ -5,7 +5,7 @@ #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "ekat/util/ekat_arch.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -21,10 +21,10 @@ namespace unit_test { * Unit-tests for p3_functions. */ template -struct UnitWrap::UnitTest::TestP3Conservation +struct UnitWrap::UnitTest::TestP3Conservation : public UnitWrap::UnitTest::Base { - static void cloud_water_conservation_tests_device() { + void cloud_water_conservation_tests_device() { using KTH = KokkosTypes; @@ -78,7 +78,7 @@ struct UnitWrap::UnitTest::TestP3Conservation REQUIRE(cwdc_host[0].qc2qr_autoconv_tend * cwdc[0].dt <= cwdc_host[0].qc); } - static void rain_water_conservation_tests_device() { + void rain_water_conservation_tests_device() { using KTH = KokkosTypes; RainWaterConservationData rwdc[1] = {{sp(1e-5), 0.0, 0.0, 0.0, 0.0, sp(1.1), sp(1e-4), 0.0, 0.0 }}; @@ -131,7 +131,7 @@ struct UnitWrap::UnitTest::TestP3Conservation REQUIRE( rwdc_host(0).qr2qv_evap_tend * rwdc_host(0).dt <= rwdc_host(0).qr); } - static void ice_water_conservation_tests_device(){ + void ice_water_conservation_tests_device() { using KTH = KokkosTypes; IceWaterConservationData iwdc[1] = {{sp(1e-5), 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, sp(1.1), sp(1e-4), 0.0}}; @@ -173,7 +173,7 @@ struct UnitWrap::UnitTest::TestP3Conservation } - static void run() + void run() { cloud_water_conservation_tests_device(); @@ -182,7 +182,7 @@ struct UnitWrap::UnitTest::TestP3Conservation ice_water_conservation_tests_device(); } - static void cloud_water_conservation_unit_bfb_tests(){ + void cloud_water_conservation_unit_bfb_tests() { using KTH = KokkosTypes; @@ -222,9 +222,11 @@ struct UnitWrap::UnitTest::TestP3Conservation std::copy(&cwdc[0], &cwdc[0] + max_pack_size, cwdc_host.data()); Kokkos::deep_copy(cwdc_device, cwdc_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - cloud_water_conservation(cwdc[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + cwdc[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -263,7 +265,7 @@ struct UnitWrap::UnitTest::TestP3Conservation Kokkos::deep_copy(cwdc_host, cwdc_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(cwdc[s].qc == cwdc_host(s).qc); REQUIRE(cwdc[s].qc2qr_autoconv_tend == cwdc_host(s).qc2qr_autoconv_tend); @@ -275,9 +277,14 @@ struct UnitWrap::UnitTest::TestP3Conservation REQUIRE(cwdc[s].qv2qi_vapdep_tend == cwdc_host(s).qv2qi_vapdep_tend); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + cwdc_host(s).write(Base::m_fid); + } + } } - static void ice_water_conservation_unit_bfb_tests() + void ice_water_conservation_unit_bfb_tests() { using KTH = KokkosTypes; @@ -312,9 +319,11 @@ struct UnitWrap::UnitTest::TestP3Conservation std::copy(&iwdc[0], &iwdc[0] + max_pack_size, iwdc_host.data()); Kokkos::deep_copy(iwdc_device, iwdc_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - ice_water_conservation(iwdc[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + iwdc[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -356,7 +365,7 @@ struct UnitWrap::UnitTest::TestP3Conservation Kokkos::deep_copy(iwdc_host, iwdc_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(iwdc[s].qi == iwdc_host(s).qi); REQUIRE(iwdc[s].qv2qi_vapdep_tend == iwdc_host(s).qv2qi_vapdep_tend ); @@ -370,9 +379,14 @@ struct UnitWrap::UnitTest::TestP3Conservation REQUIRE(iwdc[s].qi2qr_melt_tend == iwdc_host(s).qi2qr_melt_tend); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + iwdc_host(s).write(Base::m_fid); + } + } } - static void rain_water_conservation_unit_bfb_tests(){ + void rain_water_conservation_unit_bfb_tests() { using KTH = KokkosTypes; @@ -407,9 +421,11 @@ struct UnitWrap::UnitTest::TestP3Conservation std::copy(&rwdc[0], &rwdc[0] + max_pack_size, rwdc_host.data()); Kokkos::deep_copy(rwdc_device, rwdc_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - rain_water_conservation(rwdc[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + rwdc[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -448,7 +464,7 @@ struct UnitWrap::UnitTest::TestP3Conservation Kokkos::deep_copy(rwdc_host, rwdc_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(rwdc[s].qr == rwdc_host(s).qr); REQUIRE(rwdc[s].qc2qr_autoconv_tend == rwdc_host(s).qc2qr_autoconv_tend); @@ -460,9 +476,14 @@ struct UnitWrap::UnitTest::TestP3Conservation REQUIRE(rwdc[s].qr2qi_immers_freeze_tend == rwdc_host(s).qr2qi_immers_freeze_tend); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + rwdc_host(s).write(Base::m_fid); + } + } } - static void run_bfb() { + void run_bfb() { cloud_water_conservation_unit_bfb_tests(); rain_water_conservation_unit_bfb_tests(); @@ -473,9 +494,9 @@ struct UnitWrap::UnitTest::TestP3Conservation }; template -struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce +struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce : public UnitWrap::UnitTest::Base { - static void update_prognostic_ice_unit_bfb_tests() { + void update_prognostic_ice_unit_bfb_tests() { constexpr Scalar nmltratio = C::nmltratio; constexpr Scalar dt = 1.8000E+03; @@ -483,7 +504,7 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; - //fortran generated data is input to the following + //baseline generated data is input to the following P3UpdatePrognosticIceData pupidc[max_pack_size] = { {4.9078E-19, 1.5312E-09, 4.4387E-09, 3.7961E+06, 1.7737E-04, 0.0000E+00, 3.8085E-08, 5.1281E+04, 1.9251E-15, @@ -591,9 +612,11 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce std::copy(&pupidc[0], &pupidc[0] + max_pack_size, pupidc_host.data()); Kokkos::deep_copy(pupidc_device, pupidc_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - update_prognostic_ice(pupidc[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + pupidc[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -679,7 +702,7 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce Kokkos::deep_copy(pupidc_host, pupidc_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(pupidc[s].th_atm == pupidc_host(s).th_atm); REQUIRE(pupidc[s].qc == pupidc_host(s).qc); @@ -693,22 +716,27 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce REQUIRE(pupidc[s].bm == pupidc_host(s).bm ); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + pupidc_host(s).write(Base::m_fid); + } + } } - static void run_bfb(){ + void run_bfb() { update_prognostic_ice_unit_bfb_tests(); } }; //TestP3UpdatePrognosticIce template -struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables +struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables : public UnitWrap::UnitTest::Base { - static void get_time_space_phys_variables_unit_bfb_tests(){ + void get_time_space_phys_variables_unit_bfb_tests() { constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; - //fortran generated data is input to the following + //baseline generated data is input to the following GetTimeSpacePhysVarsData gtspvd[max_pack_size] = { // T_atm, pres, rho, latent_heat_vapor, latent_heat_sublim, qv_sat_l, qv_sat_i {2.9792E+02, 9.8711E+04, 1.1532E+00, latvap, latvap+latice, 2.0321E-02, 2.0321E-02}, @@ -737,9 +765,11 @@ struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables std::copy(>spvd[0], >spvd[0] + max_pack_size, gtspvd_host.data()); Kokkos::deep_copy(gtspvd_device, gtspvd_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - get_time_space_phys_variables(gtspvd[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + gtspvd[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -794,7 +824,7 @@ struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables Kokkos::deep_copy(gtspvd_host, gtspvd_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(gtspvd[s].mu == gtspvd_host(s).mu); REQUIRE(gtspvd[s].dv == gtspvd_host(s).dv); @@ -807,20 +837,25 @@ struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables REQUIRE(gtspvd[s].eii == gtspvd_host(s).eii); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + gtspvd_host(s).write(Base::m_fid); + } + } } - static void run_bfb(){ + void run_bfb() { get_time_space_phys_variables_unit_bfb_tests(); } }; //TestGetTimeSpacePhysVariables template -struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq +struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq : public UnitWrap::UnitTest::Base { - static void update_prognostic_liquid_unit_bfb_tests(){ + void update_prognostic_liquid_unit_bfb_tests() { constexpr Scalar latvap = C::LatVap; - //fortran generated data is input to the following + //baseline generated data is input to the following P3UpdatePrognosticLiqData pupldc[max_pack_size] = { {1.0631E-12, 1.0631E+00, 1.5833E-12, 1.5833E+00, 2.4190E-02, 0.0000E+00, 0.0000E+00, 0.0000E+00, 4.2517E+00, @@ -896,9 +931,11 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq std::copy(&pupldc[0], &pupldc[0] + max_pack_size, pupldc_host.data()); Kokkos::deep_copy(pupldc_device, pupldc_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - update_prognostic_liquid(pupldc[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + pupldc[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -971,7 +1008,7 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq Kokkos::deep_copy(pupldc_host, pupldc_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(pupldc[s].th_atm == pupldc_host(s).th_atm); REQUIRE(pupldc[s].qv == pupldc_host(s).qv); @@ -981,18 +1018,23 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq REQUIRE(pupldc[s].nr == pupldc_host(s).nr); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + pupldc_host(s).write(Base::m_fid); + } + } } - static void run_bfb(){ + void run_bfb() { update_prognostic_liquid_unit_bfb_tests(); } }; //TestP3UpdatePrognosticLiq template -struct UnitWrap::UnitTest::TestP3FunctionsImposeMaxTotalNi +struct UnitWrap::UnitTest::TestP3FunctionsImposeMaxTotalNi : public UnitWrap::UnitTest::Base { - static void impose_max_total_ni_bfb_test(){ + void impose_max_total_ni_bfb_test() { constexpr Scalar max_total_ni = 740.0e3; ImposeMaxTotalNiData dc[max_pack_size]= { @@ -1026,9 +1068,11 @@ struct UnitWrap::UnitTest::TestP3FunctionsImposeMaxTotalNi std::copy(&dc[0], &dc[0] + max_pack_size, dc_host.data()); Kokkos::deep_copy(dc_device, dc_host); - //Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - impose_max_total_ni(dc[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + dc[i].read(Base::m_fid); + } } //Run function from a kernal and copy results back to the host @@ -1054,15 +1098,20 @@ struct UnitWrap::UnitTest::TestP3FunctionsImposeMaxTotalNi Kokkos::deep_copy(dc_host, dc_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(dc[s].ni_local == dc_host(s).ni_local); REQUIRE(dc[s].inv_rho_local == dc_host(s).inv_rho_local); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + dc_host(s).write(Base::m_fid); + } + } } - static void run_bfb(){ + void run_bfb() { impose_max_total_ni_bfb_test(); } @@ -1075,24 +1124,39 @@ struct UnitWrap::UnitTest::TestP3FunctionsImposeMaxTotalNi namespace { TEST_CASE("p3_conservation_test", "[p3_unit_tests]"){ - scream::p3::unit_test::UnitWrap::UnitTest::TestP3Conservation::run(); - scream::p3::unit_test::UnitWrap::UnitTest::TestP3Conservation::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestP3Conservation; + + T t; + t.run(); + t.run_bfb(); } TEST_CASE("p3_get_time_space_phys_variables_test", "[p3_unit_tests]"){ - scream::p3::unit_test::UnitWrap::UnitTest::TestGetTimeSpacePhysVariables::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestGetTimeSpacePhysVariables; + + T t; + t.run_bfb(); } TEST_CASE("p3_update_prognostic_ice_test", "[p3_unit_tests]"){ - scream::p3::unit_test::UnitWrap::UnitTest::TestP3UpdatePrognosticIce::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestP3UpdatePrognosticIce; + + T t; + t.run_bfb(); } TEST_CASE("p3_update_prognostic_liquid_test", "[p3_unit_tests]"){ - scream::p3::unit_test::UnitWrap::UnitTest::TestP3UpdatePrognosticLiq::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestP3UpdatePrognosticLiq; + + T t; + t.run_bfb(); } TEST_CASE("p3_impose_max_total_ni_test", "[p3_unit_tests]"){ - scream::p3::unit_test::UnitWrap::UnitTest::TestP3FunctionsImposeMaxTotalNi::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestP3FunctionsImposeMaxTotalNi; + + T t; + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_upwind_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_upwind_unit_tests.cpp index fde2ca644cbf..6e83a8ee4eaa 100644 --- a/components/eamxx/src/physics/p3/tests/p3_upwind_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_upwind_unit_tests.cpp @@ -3,11 +3,9 @@ #include "p3_unit_tests_common.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "share/scream_types.hpp" -#include "share/util/scream_setup_random_test.hpp" -#include "share/util/scream_setup_random_test.hpp" #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" @@ -36,9 +34,9 @@ namespace unit_test { // cells in the domain are 0. This lets us check the restricted-domain usage of // the upwind routine in the first time step. template -struct UnitWrap::UnitTest::TestUpwind { +struct UnitWrap::UnitTest::TestUpwind : public UnitWrap::UnitTest::Base { -static void run_phys() +void run_phys() { using ekat::repack; constexpr auto SPS = SCREAM_SMALL_PACK_SIZE; @@ -201,11 +199,12 @@ static void run_phys() } } -static void run_bfb() +void run_bfb() { - auto engine = setup_random_test(); + // With stored baselines, we must use a fixed seed! + auto engine = Base::get_engine(); - CalcUpwindData cuds_fortran[] = { + CalcUpwindData cuds_baseline[] = { // kts, kte, kdir, kbot, k_qxtop, na, dt_sub, CalcUpwindData( 1, 72, -1, 72, 36, 2, 1.833E+03), CalcUpwindData( 1, 72, 1, 36, 72, 2, 1.833E+03), @@ -216,28 +215,30 @@ static void run_bfb() CalcUpwindData( 1, 32, -1, 21, 7, 1, 1.833E+03), }; - static constexpr Int num_runs = sizeof(cuds_fortran) / sizeof(CalcUpwindData); + static constexpr Int num_runs = sizeof(cuds_baseline) / sizeof(CalcUpwindData); // Set up random input data - for (auto& d : cuds_fortran) { + for (auto& d : cuds_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state CalcUpwindData cuds_cxx[num_runs] = { - CalcUpwindData(cuds_fortran[0]), - CalcUpwindData(cuds_fortran[1]), - CalcUpwindData(cuds_fortran[2]), - CalcUpwindData(cuds_fortran[3]), - CalcUpwindData(cuds_fortran[4]), - CalcUpwindData(cuds_fortran[5]), - CalcUpwindData(cuds_fortran[6]), + CalcUpwindData(cuds_baseline[0]), + CalcUpwindData(cuds_baseline[1]), + CalcUpwindData(cuds_baseline[2]), + CalcUpwindData(cuds_baseline[3]), + CalcUpwindData(cuds_baseline[4]), + CalcUpwindData(cuds_baseline[5]), + CalcUpwindData(cuds_baseline[6]), }; - // Get data from fortran - for (auto& d : cuds_fortran) { - calc_first_order_upwind_step(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : cuds_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx @@ -245,23 +246,23 @@ static void run_bfb() for (auto& d : cuds_cxx) { Real** fluxes, **vs, **qnx; d.convert_to_ptr_arr(tmp1, fluxes, vs, qnx); - calc_first_order_upwind_step_f( + calc_first_order_upwind_step_host( d.kts, d.kte, d.kdir, d.kbot, d.k_qxtop, d.dt_sub, d.rho, d.inv_rho, d.inv_dz, d.num_arrays, fluxes, vs, qnx); } - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { // Due to pack issues, we must restrict checks to the active k space - Int start = std::min(cuds_fortran[i].kbot, cuds_fortran[i].k_qxtop) - 1; // 0-based indx - Int end = std::max(cuds_fortran[i].kbot, cuds_fortran[i].k_qxtop); // 0-based indx + Int start = std::min(cuds_baseline[i].kbot, cuds_baseline[i].k_qxtop) - 1; // 0-based indx + Int end = std::max(cuds_baseline[i].kbot, cuds_baseline[i].k_qxtop); // 0-based indx Real** fluxesf90, **vsf90, **qnxf90, **fluxescxx, **vscxx, **qnxcxx; - cuds_fortran[i].convert_to_ptr_arr(tmp1, fluxesf90, vsf90, qnxf90); + cuds_baseline[i].convert_to_ptr_arr(tmp1, fluxesf90, vsf90, qnxf90); cuds_cxx[i].convert_to_ptr_arr(tmp1, fluxescxx, vscxx, qnxcxx); - for (int n = 0; n < cuds_fortran[i].num_arrays; ++n) { + for (int n = 0; n < cuds_baseline[i].num_arrays; ++n) { for (Int k = start; k < end; ++k) { REQUIRE(fluxesf90[n][k] == fluxescxx[n][k]); REQUIRE(qnxf90[n][k] == qnxcxx[n][k]); @@ -269,23 +270,29 @@ static void run_bfb() } } } + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + cuds_cxx[i].write(Base::m_fid); + } + } } }; template -struct UnitWrap::UnitTest::TestGenSed { +struct UnitWrap::UnitTest::TestGenSed : public UnitWrap::UnitTest::Base { -static void run_phys() +void run_phys() { // TODO } -static void run_bfb() +void run_bfb() { - auto engine = setup_random_test(); + // With stored baselines, we must use a fixed seed! + auto engine = Base::get_engine(); - GenSedData gsds_fortran[] = { + GenSedData gsds_baseline[] = { // kts, kte, kdir, k_qxtop, k_qxbot, kbot, Co_max, dt_left, prt_accum, num_arrays GenSedData(1, 72, -1, 36, 72, 72, 9.196E-02, 1.818E+01, 4.959E-05, 2), GenSedData(1, 72, -1, 36, 57, 72, 4.196E-01, 1.418E+02, 4.959E-06, 1), @@ -293,25 +300,27 @@ static void run_bfb() GenSedData(1, 72, -1, 72, 72, 72, 4.196E-01, 1.418E+02, 4.959E-06, 1), }; - static constexpr Int num_runs = sizeof(gsds_fortran) / sizeof(GenSedData); + static constexpr Int num_runs = sizeof(gsds_baseline) / sizeof(GenSedData); // Set up random input data - for (auto& d : gsds_fortran) { + for (auto& d : gsds_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state GenSedData gsds_cxx[num_runs] = { - GenSedData(gsds_fortran[0]), - GenSedData(gsds_fortran[1]), - GenSedData(gsds_fortran[2]), - GenSedData(gsds_fortran[3]), + GenSedData(gsds_baseline[0]), + GenSedData(gsds_baseline[1]), + GenSedData(gsds_baseline[2]), + GenSedData(gsds_baseline[3]), }; - // Get data from fortran - for (auto& d : gsds_fortran) { - generalized_sedimentation(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : gsds_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx @@ -319,31 +328,36 @@ static void run_bfb() for (auto& d : gsds_cxx) { Real** fluxes, **vs, **qnx; d.convert_to_ptr_arr(tmp1, fluxes, vs, qnx); - generalized_sedimentation_f(d.kts, d.kte, d.kdir, d.k_qxtop, + generalized_sedimentation_host(d.kts, d.kte, d.kdir, d.k_qxtop, &d.k_qxbot, d.kbot, d.Co_max, &d.dt_left, &d.prt_accum, d.inv_dz, d.inv_rho, d.rho, d.num_arrays, fluxes, vs, qnx); } - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { // Due to pack issues, we must restrict checks to the active k space - Int start = std::min(gsds_fortran[i].k_qxbot, gsds_fortran[i].k_qxtop) - 1; // 0-based indx - Int end = std::max(gsds_fortran[i].k_qxbot, gsds_fortran[i].k_qxtop); // 0-based indx + Int start = std::min(gsds_baseline[i].k_qxbot, gsds_baseline[i].k_qxtop) - 1; // 0-based indx + Int end = std::max(gsds_baseline[i].k_qxbot, gsds_baseline[i].k_qxtop); // 0-based indx Real** fluxesf90, **vsf90, **qnxf90, **fluxescxx, **vscxx, **qnxcxx; - gsds_fortran[i].convert_to_ptr_arr(tmp1, fluxesf90, vsf90, qnxf90); + gsds_baseline[i].convert_to_ptr_arr(tmp1, fluxesf90, vsf90, qnxf90); gsds_cxx[i].convert_to_ptr_arr(tmp1, fluxescxx, vscxx, qnxcxx); - for (int n = 0; n < gsds_fortran[i].num_arrays; ++n) { + for (int n = 0; n < gsds_baseline[i].num_arrays; ++n) { for (Int k = start; k < end; ++k) { REQUIRE(fluxesf90[n][k] == fluxescxx[n][k]); REQUIRE(qnxf90[n][k] == qnxcxx[n][k]); } } - REQUIRE(gsds_fortran[i].k_qxbot == gsds_cxx[i].k_qxbot); - REQUIRE(gsds_fortran[i].dt_left == gsds_cxx[i].dt_left); - REQUIRE(gsds_fortran[i].prt_accum == gsds_cxx[i].prt_accum); + REQUIRE(gsds_baseline[i].k_qxbot == gsds_cxx[i].k_qxbot); + REQUIRE(gsds_baseline[i].dt_left == gsds_cxx[i].dt_left); + REQUIRE(gsds_baseline[i].prt_accum == gsds_cxx[i].prt_accum); + } + } + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + gsds_cxx[i].write(Base::m_fid); } } } @@ -358,18 +372,20 @@ namespace { TEST_CASE("p3_upwind", "[p3_functions]") { - using TU = scream::p3::unit_test::UnitWrap::UnitTest::TestUpwind; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestUpwind; - TU::run_phys(); - TU::run_bfb(); + T t; + t.run_phys(); + t.run_bfb(); } TEST_CASE("p3_gen_sed", "[p3_functions]") { - using TG = scream::p3::unit_test::UnitWrap::UnitTest::TestGenSed; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestGenSed; - TG::run_phys(); - TG::run_bfb(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/register_physics.hpp b/components/eamxx/src/physics/register_physics.hpp index d837e96311f4..dc1ce5745d38 100644 --- a/components/eamxx/src/physics/register_physics.hpp +++ b/components/eamxx/src/physics/register_physics.hpp @@ -41,6 +41,9 @@ #ifdef EAMXX_HAS_ML_CORRECTION #include "physics/ml_correction/eamxx_ml_correction_process_interface.hpp" #endif +#ifdef EAMXX_HAS_IOP_FORCING +#include "physics/iop_forcing/eamxx_iop_forcing_process_interface.hpp" +#endif namespace scream { @@ -65,7 +68,7 @@ inline void register_physics () { proc_factory.register_product("Nudging",&create_atmosphere_process); #endif #ifdef EAMXX_HAS_MAM - proc_factory.register_product("mam4_micro",&create_atmosphere_process); + proc_factory.register_product("mam4_aero_microphys",&create_atmosphere_process); proc_factory.register_product("mam4_optics",&create_atmosphere_process); proc_factory.register_product("mam4_drydep",&create_atmosphere_process); proc_factory.register_product("mam4_aci",&create_atmosphere_process); @@ -82,6 +85,9 @@ inline void register_physics () { #ifdef EAMXX_HAS_ML_CORRECTION proc_factory.register_product("MLCorrection",&create_atmosphere_process); #endif +#ifdef EAMXX_HAS_IOP_FORCING + proc_factory.register_product("iop_forcing",&create_atmosphere_process); +#endif // If no physics was enabled, silence compile warning about unused var (void) proc_factory; diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 75082e789ed6..0121741963d9 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -99,16 +99,16 @@ void RRTMGPRadiation::set_grids(const std::shared_ptr grids_ m_ncol = m_grid->get_num_local_dofs(); m_nlay = m_grid->get_num_vertical_levels(); - if (m_iop) { + if (m_iop_data_manager) { // For IOP runs, we need to use the lat/lon from the // IOP files instead of the geometry data. Deep copy // to device and sync to host since both will be used. m_lat = m_grid->get_geometry_data("lat").clone(); - m_lat.deep_copy(m_iop->get_params().get("target_latitude")); + m_lat.deep_copy(m_iop_data_manager->get_params().get("target_latitude")); m_lat.sync_to_host(); m_lon = m_grid->get_geometry_data("lon").clone(); - m_lon.deep_copy(m_iop->get_params().get("target_longitude")); + m_lon.deep_copy(m_iop_data_manager->get_params().get("target_longitude")); m_lon.sync_to_host(); } else { m_lat = m_grid->get_geometry_data("lat"); @@ -616,9 +616,13 @@ void RRTMGPRadiation::initialize_impl(const RunType /* run_type */) { // for duration of simulation. m_orbital_year = m_params.get("orbital_year",-9999); // Get orbital parameters from yaml file - m_orbital_eccen = m_params.get("orbital_eccentricity",-9999); - m_orbital_obliq = m_params.get("orbital_obliquity" ,-9999); - m_orbital_mvelp = m_params.get("orbital_mvelp" ,-9999); + m_orbital_eccen = m_params.get("orbital_eccentricity",-9999); + m_orbital_obliq = m_params.get("orbital_obliquity" ,-9999); + m_orbital_mvelp = m_params.get("orbital_mvelp" ,-9999); + + // Value for prescribing an invariant solar constant (i.e. total solar irradiance at + // TOA). Used for idealized experiments such as RCE. Disabled when value is less than 0. + m_fixed_total_solar_irradiance = m_params.get("fixed_total_solar_irradiance", -9999); // Determine whether or not we are using a fixed solar zenith angle (positive value) m_fixed_solar_zenith_angle = m_params.get("Fixed Solar Zenith Angle", -9999); @@ -840,6 +844,12 @@ void RRTMGPRadiation::run_impl (const double dt) { shr_orb_decl_c2f(calday, eccen, mvelpp, lambm0, obliqr, &delta, &eccf); + // Overwrite eccf if using a fixed solar constant. + auto fixed_total_solar_irradiance = m_fixed_total_solar_irradiance; + if (fixed_total_solar_irradiance >= 0){ + eccf = fixed_total_solar_irradiance/1360.9; + } + // Precompute VMR for all gases, on all cols, before starting the chunks loop // // h2o is taken from qv diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp index 329c7eeaba8d..02a047cce598 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp @@ -95,6 +95,11 @@ class RRTMGPRadiation : public AtmosphereProcess { Real m_orbital_obliq; // Obliquity Real m_orbital_mvelp; // Vernal Equinox Mean Longitude of Perihelion + // Value for prescribing an invariant solar constant (i.e. total solar irradiance + // at TOA). Used for idealized experiments such as RCE. This is only used when a + // positive value is supplied. + Real m_fixed_total_solar_irradiance; + // Fixed solar zenith angle to use for shortwave calculations // This is only used if a positive value is supplied Real m_fixed_solar_zenith_angle; diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index 216722b3766b..e807643b05a9 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -167,9 +167,6 @@ using MDRP = typename conv::MDRP; template using view_t = Kokkos::View; -template -using oview_t = Kokkos::Experimental::OffsetView; - template using hview_t = Kokkos::View; @@ -243,7 +240,7 @@ static void rrtmgp_initialize( load_cld_lutcoeff(cloud_optics_lw_k, cloud_optics_file_lw); // initialize kokkos rrtmgp pool allocator - const size_t base_ref = 18000; + const size_t base_ref = 80000; const size_t ncol = gas_concs.ncol; const size_t nlay = gas_concs.nlay; const size_t nlev = SCREAM_NUM_VERTICAL_LEV; @@ -561,9 +558,9 @@ static void rrtmgp_sw( const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { // Get problem sizes - int nbnd = k_dist.get_nband(); - int ngpt = k_dist.get_ngpt(); - int ngas = gas_concs.get_num_gases(); + const int nbnd = k_dist.get_nband(); + const int ngpt = k_dist.get_ngpt(); + const int ngas = gas_concs.get_num_gases(); // Associate local pointers for fluxes auto &flux_up = fluxes.flux_up; @@ -604,7 +601,7 @@ static void rrtmgp_sw( }); // Get daytime indices - auto dayIndices = view_t("dayIndices", ncol); + auto dayIndices = pool_t::template alloc_and_init(ncol); Kokkos::deep_copy(dayIndices, -1); int nday = 0; @@ -619,24 +616,59 @@ static void rrtmgp_sw( if (nday == 0) { // No daytime columns in this chunk, skip the rest of this routine + pool_t::dealloc(dayIndices); return; } + // Allocate temporaries from pool + const int size1 = nday; + const int size2 = nday*nlay; // 4 + const int size3 = nday*(nlay+1); // 5 + const int size4 = ncol*nlay; + const int size5 = nbnd*nday; //2 + const int size6 = nday*ngpt; + const int size7 = nday*(nlay+1)*nbnd; // 3 + const int size8 = ncol*nlay*(k_dist.get_ngas()+1); + + const int total_size = size1 + size2*4 + size3*5 + size4 + size5*2 + size6 + size7*3 + size8; + auto data = pool_t::template alloc_and_init(total_size); RealT* dcurr = data.data(); + + auto mu0_day = view_t (dcurr, nday); dcurr += size1; + + auto p_lay_day = view_t (dcurr, nday, nlay); dcurr += size2; + auto t_lay_day = view_t (dcurr, nday, nlay); dcurr += size2; + auto vmr_day = view_t (dcurr, nday, nlay); dcurr += size2; + auto t_lay_limited = view_t (dcurr, nday, nlay); dcurr += size2; + + auto p_lev_day = view_t (dcurr, nday, nlay+1); dcurr += size3; + auto t_lev_day = view_t (dcurr, nday, nlay+1); dcurr += size3; + auto flux_up_day = view_t (dcurr, nday, nlay+1); dcurr += size3; + auto flux_dn_day = view_t (dcurr, nday, nlay+1); dcurr += size3; + auto flux_dn_dir_day = view_t (dcurr, nday, nlay+1); dcurr += size3; + + auto vmr = view_t (dcurr, ncol, nlay); dcurr += size4; + + auto sfc_alb_dir_T = view_t (dcurr, nbnd, nday); dcurr += size5; + auto sfc_alb_dif_T = view_t (dcurr, nbnd, nday); dcurr += size5; + + auto toa_flux = view_t (dcurr, nday, ngpt); dcurr += size6; + + auto bnd_flux_up_day = view_t(dcurr, nday, nlay+1, nbnd); dcurr += size7; + auto bnd_flux_dn_day = view_t(dcurr, nday, nlay+1, nbnd); dcurr += size7; + auto bnd_flux_dn_dir_day = view_t(dcurr, nday, nlay+1, nbnd); dcurr += size7; + + auto col_gas = view_t(dcurr, ncol, nlay, k_dist.get_ngas()+1); dcurr += size8; + // Subset mu0 - auto mu0_day = view_t("mu0_day", nday); Kokkos::parallel_for(nday, KOKKOS_LAMBDA(int iday) { mu0_day(iday) = mu0(dayIndices(iday)); }); // subset state variables - auto p_lay_day = view_t("p_lay_day", nday, nlay); - auto t_lay_day = view_t("t_lay_day", nday, nlay); Kokkos::parallel_for(MDRP::template get<2>({nlay,nday}), KOKKOS_LAMBDA(int ilay, int iday) { p_lay_day(iday,ilay) = p_lay(dayIndices(iday),ilay); t_lay_day(iday,ilay) = t_lay(dayIndices(iday),ilay); }); - auto p_lev_day = view_t("p_lev_day", nday, nlay+1); - auto t_lev_day = view_t("t_lev_day", nday, nlay+1); Kokkos::parallel_for(MDRP::template get<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { p_lev_day(iday,ilev) = p_lev(dayIndices(iday),ilev); t_lev_day(iday,ilev) = t_lev(dayIndices(iday),ilev); @@ -647,8 +679,6 @@ static void rrtmgp_sw( gas_concs_t gas_concs_day; gas_concs_day.init(gas_names, nday, nlay); for (int igas = 0; igas < ngas; igas++) { - auto vmr_day = view_t("vmr_day", nday, nlay); - auto vmr = view_t("vmr" , ncol, nlay); gas_concs.get_vmr(gas_names[igas], vmr); Kokkos::parallel_for(MDRP::template get<2>({nlay,nday}), KOKKOS_LAMBDA(int ilay, int iday) { vmr_day(iday,ilay) = vmr(dayIndices(iday),ilay); @@ -680,20 +710,12 @@ static void rrtmgp_sw( // RRTMGP assumes surface albedos have a screwy dimension ordering // for some strange reason, so we need to transpose these; also do // daytime subsetting in the same kernel - view_t sfc_alb_dir_T("sfc_alb_dir", nbnd, nday); - view_t sfc_alb_dif_T("sfc_alb_dif", nbnd, nday); Kokkos::parallel_for(MDRP::template get<2>({nbnd,nday}), KOKKOS_LAMBDA(int ibnd, int icol) { sfc_alb_dir_T(ibnd,icol) = sfc_alb_dir(dayIndices(icol),ibnd); sfc_alb_dif_T(ibnd,icol) = sfc_alb_dif(dayIndices(icol),ibnd); }); // Temporaries we need for daytime-only fluxes - auto flux_up_day = view_t("flux_up_day", nday, nlay+1); - auto flux_dn_day = view_t("flux_dn_day", nday, nlay+1); - auto flux_dn_dir_day = view_t("flux_dn_dir_day", nday, nlay+1); - auto bnd_flux_up_day = view_t("bnd_flux_up_day", nday, nlay+1, nbnd); - auto bnd_flux_dn_day = view_t("bnd_flux_dn_day", nday, nlay+1, nbnd); - auto bnd_flux_dn_dir_day = view_t("bnd_flux_dn_dir_day", nday, nlay+1, nbnd); fluxes_t fluxes_day; fluxes_day.flux_up = flux_up_day; fluxes_day.flux_dn = flux_dn_day; @@ -713,18 +735,14 @@ static void rrtmgp_sw( } // Limit temperatures for gas optics look-up tables - auto t_lay_limited = view_t("t_lay_limited", nday, nlay); limit_to_bounds_k(t_lay_day, k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), t_lay_limited); // Do gas optics - view_t toa_flux("toa_flux", nday, ngpt); bool top_at_1 = false; Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, bool& val) { val |= p_lay(0, 0) < p_lay(0, nlay-1); }, Kokkos::LOr(top_at_1)); - oview_t col_gas("col_gas", std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); - k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, col_gas, optics, toa_flux); if (extra_clnsky_diag) { k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, col_gas, optics_no_aerosols, toa_flux); @@ -803,6 +821,9 @@ static void rrtmgp_sw( clnsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); }); } + + pool_t::dealloc(data); + pool_t::dealloc(dayIndices); } /* @@ -819,6 +840,25 @@ static void rrtmgp_lw( { // Problem size int nbnd = k_dist.get_nband(); + int constexpr max_gauss_pts = 4; + + const int size1 = ncol; + const int size2 = nbnd*ncol; + const int size3 = max_gauss_pts*max_gauss_pts; + const int size4 = ncol*nlay; + const int size5 = ncol*(nlay+1); + const int size6 = ncol*nlay*(k_dist.get_ngas()+1); + + const int total_size = size1 + size2 + size3*2 + size4 + size5 + size6; + auto data = pool_t::template alloc_and_init(total_size); RealT *dcurr = data.data(); + + view_t t_sfc (dcurr, ncol); dcurr += size1; + view_t emis_sfc (dcurr, nbnd,ncol); dcurr += size2; + view_t gauss_Ds (dcurr, max_gauss_pts,max_gauss_pts); dcurr += size3; + view_t gauss_wts (dcurr, max_gauss_pts,max_gauss_pts); dcurr += size3; + view_t t_lay_limited(dcurr, ncol, nlay); dcurr += size4; + view_t t_lev_limited(dcurr, ncol, nlay+1); dcurr += size5; + view_t col_gas (dcurr, ncol, nlay, k_dist.get_ngas()+1); dcurr += size6; // Associate local pointers for fluxes auto &flux_up = fluxes.flux_up; @@ -863,8 +903,6 @@ static void rrtmgp_lw( // Boundary conditions source_func_t lw_sources; lw_sources.alloc(ncol, nlay, k_dist); - view_t t_sfc ("t_sfc" ,ncol); - view_t emis_sfc("emis_sfc",nbnd,ncol); bool top_at_1 = false; Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, bool& val) { @@ -882,32 +920,31 @@ static void rrtmgp_lw( // Weights and angle secants for first order (k=1) Gaussian quadrature. // Values from Table 2, Clough et al, 1992, doi:10.1029/92JD01419 // after Abramowitz & Stegun 1972, page 921 - int constexpr max_gauss_pts = 4; - hview_t gauss_Ds_host ("gauss_Ds" ,max_gauss_pts,max_gauss_pts); - gauss_Ds_host(0,0) = 1.66 ; gauss_Ds_host(1,0) = 0.; gauss_Ds_host(2,0) = 0.; gauss_Ds_host(3,0) = 0.; - gauss_Ds_host(0,1) = 1.18350343; gauss_Ds_host(1,1) = 2.81649655; gauss_Ds_host(2,1) = 0.; gauss_Ds_host(3,1) = 0.; - gauss_Ds_host(0,2) = 1.09719858; gauss_Ds_host(1,2) = 1.69338507; gauss_Ds_host(2,2) = 4.70941630; gauss_Ds_host(3,2) = 0.; - gauss_Ds_host(0,3) = 1.06056257; gauss_Ds_host(1,3) = 1.38282560; gauss_Ds_host(2,3) = 2.40148179; gauss_Ds_host(3,3) = 7.15513024; - - hview_t gauss_wts_host("gauss_wts",max_gauss_pts,max_gauss_pts); - gauss_wts_host(0,0) = 0.5 ; gauss_wts_host(1,0) = 0. ; gauss_wts_host(2,0) = 0. ; gauss_wts_host(3,0) = 0. ; - gauss_wts_host(0,1) = 0.3180413817; gauss_wts_host(1,1) = 0.1819586183; gauss_wts_host(2,1) = 0. ; gauss_wts_host(3,1) = 0. ; - gauss_wts_host(0,2) = 0.2009319137; gauss_wts_host(1,2) = 0.2292411064; gauss_wts_host(2,2) = 0.0698269799; gauss_wts_host(3,2) = 0. ; - gauss_wts_host(0,3) = 0.1355069134; gauss_wts_host(1,3) = 0.2034645680; gauss_wts_host(2,3) = 0.1298475476; gauss_wts_host(3,3) = 0.0311809710; - - view_t gauss_Ds ("gauss_Ds" ,max_gauss_pts,max_gauss_pts); - view_t gauss_wts("gauss_wts",max_gauss_pts,max_gauss_pts); + RealT gauss_Ds_host_raw[max_gauss_pts][max_gauss_pts] = { + {1.66, 1.18350343, 1.09719858, 1.06056257}, + {0., 2.81649655, 1.69338507, 1.38282560}, + {0., 0., 4.70941630, 2.40148179}, + {0., 0., 0., 7.15513024} + }; + hview_t gauss_Ds_host (&gauss_Ds_host_raw[0][0], max_gauss_pts, max_gauss_pts); + + RealT gauss_wts_host_raw[max_gauss_pts][max_gauss_pts] = { + {0.5, 0.3180413817, 0.2009319137, 0.1355069134}, + {0., 0.1819586183, 0.2292411064, 0.2034645680}, + {0., 0., 0.0698269799, 0.1298475476}, + {0., 0., 0., 0.0311809710} + }; + + hview_t gauss_wts_host(&gauss_wts_host_raw[0][0],max_gauss_pts,max_gauss_pts); + Kokkos::deep_copy(gauss_Ds, gauss_Ds_host); Kokkos::deep_copy(gauss_wts, gauss_wts_host); // Limit temperatures for gas optics look-up tables - auto t_lay_limited = view_t("t_lay_limited", ncol, nlay); - auto t_lev_limited = view_t("t_lev_limited", ncol, nlay+1); limit_to_bounds_k(t_lay, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lay_limited); limit_to_bounds_k(t_lev, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lev_limited); // Do gas optics - oview_t col_gas("col_gas", std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, col_gas, optics, lw_sources, view_t(), t_lev_limited); if (extra_clnsky_diag) { k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, col_gas, optics_no_aerosols, lw_sources, view_t(), t_lev_limited); @@ -941,23 +978,23 @@ static void rrtmgp_lw( // Compute clean-sky fluxes rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics_no_aerosols, top_at_1, lw_sources, emis_sfc, clnsky_fluxes); } + + pool_t::dealloc(data); } /* * Return a subcolumn mask consistent with a specified overlap assumption */ -static int3dk get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, const real2dk &cldf, const int overlap_option, int1dk &seeds) +template +static void get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, const CldfT &cldf, const int overlap_option, const SeedsT &seeds, const SubcT& subcolumn_mask) { - // Routine will return subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud - int3dk subcolumn_mask = int3dk("subcolumn_mask", ncol, nlay, ngpt); - // Subcolumn generators are a means for producing a variable x(i,j,k), where // // c(i,j,k) = 1 for x(i,j,k) > 1 - cldf(i,j) // c(i,j,k) = 0 for x(i,j,k) <= 1 - cldf(i,j) // // I am going to call this "cldx" to be just slightly less ambiguous - auto cldx = view_t("cldx", ncol, nlay, ngpt); + auto cldx = pool_t::template alloc_and_init(ncol, nlay, ngpt); // Apply overlap assumption to set cldx if (overlap_option == 0) { // Dummy mask, always cloudy @@ -1017,7 +1054,8 @@ static int3dk get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, subcolumn_mask(icol,ilay,igpt) = 0; } }); - return subcolumn_mask; + + pool_t::dealloc(cldx); } /* @@ -1029,7 +1067,7 @@ static void compute_cloud_area( { // Subcolumn binary cld mask; if any layers with pressure between pmin and pmax are cloudy // then 2d subcol mask is 1, otherwise it is 0 - auto subcol_mask = view_t("subcol_mask", ncol, ngpt); + auto subcol_mask = pool_t::template alloc_and_init(ncol, ngpt); Kokkos::parallel_for(MDRP::template get<3>({ngpt, nlay, ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { // NOTE: using plev would need to assume level ordering (top to bottom or bottom to top), but // using play/pmid does not @@ -1046,6 +1084,8 @@ static void compute_cloud_area( cld_area(icol) += subcol_mask(icol,igpt) * ngpt_inv; } }); + + pool_t::dealloc(subcol_mask); } /* @@ -1078,7 +1118,7 @@ static void compute_aerocom_cloudtop( Kokkos::deep_copy(eff_radius_qi_at_cldtop, 0.0); // Initialize the 1D "clear fraction" as 1 (totally clear) - auto aerocom_clr = view_t("aerocom_clr", ncol); + auto aerocom_clr = pool_t::template alloc_and_init(ncol); Kokkos::deep_copy(aerocom_clr, 1.0); // Get gravity acceleration constant from constants @@ -1151,6 +1191,8 @@ static void compute_aerocom_cloudtop( // (their products) cldfrac_tot_at_cldtop(icol) = 1.0 - aerocom_clr(icol); }); + + pool_t::dealloc(aerocom_clr); } /* @@ -1253,14 +1295,17 @@ static optical_props2_t get_cloud_optics_sw( cloud_optics.set_ice_roughness(2); // Limit effective radii to be within bounds of lookup table - auto rel_limited = view_t("rel_limited", ncol, nlay); - auto rei_limited = view_t("rei_limited", ncol, nlay); + auto rel_limited = pool_t::template alloc_and_init(ncol, nlay); + auto rei_limited = pool_t::template alloc_and_init(ncol, nlay); limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); // Calculate cloud optics cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); + pool_t::dealloc(rel_limited); + pool_t::dealloc(rei_limited); + // Return optics return clouds; } @@ -1279,14 +1324,17 @@ static optical_props1_t get_cloud_optics_lw( cloud_optics.set_ice_roughness(2); // Limit effective radii to be within bounds of lookup table - auto rel_limited = view_t("rel_limited", ncol, nlay); - auto rei_limited = view_t("rei_limited", ncol, nlay); + auto rel_limited = pool_t::template alloc_and_init(ncol, nlay); + auto rei_limited = pool_t::template alloc_and_init(ncol, nlay); limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); // Calculate cloud optics cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); + pool_t::dealloc(rel_limited); + pool_t::dealloc(rei_limited); + // Return optics return clouds; } @@ -1298,6 +1346,10 @@ static optical_props2_t get_subsampled_clouds( optical_props2_t subsampled_optics; subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); subsampled_optics.alloc_2str(ncol, nlay); + + // Subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud + auto cldmask = pool_t::template alloc_and_init(ncol, nlay, ngpt); + // Check that we do not have clouds with no optical properties; this would get corrected // when we assign optical props, but we want to use a "radiative cloud fraction" // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud @@ -1305,7 +1357,7 @@ static optical_props2_t get_subsampled_clouds( // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped // even when separated by layers with no cloud properties, when in fact those layers should be // randomly overlapped. - auto cldfrac_rad = view_t("cldfrac_rad", ncol, nlay); + auto cldfrac_rad = pool_t::template alloc_and_init(ncol, nlay); Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { if (cloud_optics.tau(icol,ilay,ibnd) > 0) { cldfrac_rad(icol,ilay) = cld(icol,ilay); @@ -1319,11 +1371,11 @@ static optical_props2_t get_subsampled_clouds( int overlap = 1; // Get unique seeds for each column that are reproducible across different MPI rank layouts; // use decimal part of pressure for this, consistent with the implementation in EAM - auto seeds = view_t("seeds", ncol); + auto seeds = pool_t::template alloc_and_init(ncol); Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { seeds(icol) = 1e9 * (p_lay(icol,nlay-1) - int(p_lay(icol,nlay-1))); }); - auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); + get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds, cldmask); // Assign optical properties to subcolumns (note this implements MCICA) auto gpoint_bands = kdist.get_gpoint_bands(); Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { @@ -1338,6 +1390,11 @@ static optical_props2_t get_subsampled_clouds( subsampled_optics.g (icol,ilay,igpt) = 0; } }); + + pool_t::dealloc(cldmask); + pool_t::dealloc(cldfrac_rad); + pool_t::dealloc(seeds); + return subsampled_optics; } @@ -1349,6 +1406,10 @@ static optical_props1_t get_subsampled_clouds( optical_props1_t subsampled_optics; subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); subsampled_optics.alloc_1scl(ncol, nlay); + + // Subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud + auto cldmask = pool_t::template alloc_and_init(ncol, nlay, ngpt); + // Check that we do not have clouds with no optical properties; this would get corrected // when we assign optical props, but we want to use a "radiative cloud fraction" // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud @@ -1356,7 +1417,7 @@ static optical_props1_t get_subsampled_clouds( // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped // even when separated by layers with no cloud properties, when in fact those layers should be // randomly overlapped. - auto cldfrac_rad = view_t("cldfrac_rad", ncol, nlay); + auto cldfrac_rad = pool_t::template alloc_and_init(ncol, nlay); Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { if (cloud_optics.tau(icol,ilay,ibnd) > 0) { cldfrac_rad(icol,ilay) = cld(icol,ilay); @@ -1367,11 +1428,11 @@ static optical_props1_t get_subsampled_clouds( // Get unique seeds for each column that are reproducible across different MPI rank layouts; // use decimal part of pressure for this, consistent with the implementation in EAM; use different // seed values for longwave and shortwave - auto seeds = view_t("seeds", ncol); + auto seeds = pool_t::template alloc_and_init(ncol); Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { seeds(icol) = 1e9 * (p_lay(icol,nlay-2) - int(p_lay(icol,nlay-2))); }); - auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); + get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds, cldmask); // Assign optical properties to subcolumns (note this implements MCICA) auto gpoint_bands = kdist.get_gpoint_bands(); Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { @@ -1382,6 +1443,11 @@ static optical_props1_t get_subsampled_clouds( subsampled_optics.tau(icol,ilay,igpt) = 0; } }); + + pool_t::dealloc(cldmask); + pool_t::dealloc(cldfrac_rad); + pool_t::dealloc(seeds); + return subsampled_optics; } diff --git a/components/eamxx/src/physics/rrtmgp/tests/CMakeLists.txt b/components/eamxx/src/physics/rrtmgp/tests/CMakeLists.txt index c6fcfae76f8d..ee8e4a383c4f 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/rrtmgp/tests/CMakeLists.txt @@ -1,14 +1,12 @@ if (SCREAM_ONLY_GENERATE_BASELINES) - # Build baseline code - add_executable(generate_baseline generate_baseline.cpp) - target_link_libraries(generate_baseline PUBLIC scream_rrtmgp rrtmgp_test_utils) - - # Generate allsky baseline with the usual cmake custom command-target pair pattern + # Generate allsky baseline # Note: these "baselines" are not to compare scream with a previous version, but # rather to compare scream::rrtmgp with raw rrtmgp. - CreateUnitTestFromExec( - rrtmgp-allsky-baseline generate_baseline + CreateUnitTest ( + rrtmgp-allsky-baseline generate_baseline.cpp + LIBS scream_rrtmgp rrtmgp_test_utils LABELS baseline_gen rrtmgp + EXCLUDE_MAIN_CPP EXE_ARGS "${SCREAM_DATA_DIR}/init/rrtmgp-allsky.nc ${SCREAM_BASELINES_DIR}/data/rrtmgp-allsky-baseline.nc" ) diff --git a/components/eamxx/src/physics/rrtmgp/tests/generate_baseline.cpp b/components/eamxx/src/physics/rrtmgp/tests/generate_baseline.cpp index 97b3887acc8f..ec27719abedf 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/generate_baseline.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/generate_baseline.cpp @@ -33,15 +33,15 @@ int main (int argc, char** argv) { auto logger = std::make_shared("",LogLevel::info,comm); // Get filenames from command line - if (argc != 3) { + if (argc < 3) { std::string msg = "Missing required inputs. Usage:\n"; msg += argv[0]; msg += " inputfile baseline\n"; logger->error(msg); return 1; } - std::string inputfile(argv[argc-2]); - std::string baseline(argv[argc-1]); + std::string inputfile(argv[1]); + std::string baseline(argv[2]); // Initialize yakl yakl::init(); diff --git a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp index 0d3f18e7d841..fedb8c3d0478 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp @@ -46,7 +46,7 @@ int run_yakl(int argc, char** argv) { logger->error(msg); return 1; } - std::string inputfile, baseline, device; + std::string inputfile, baseline; for (int i = 1; i < argc-1; ++i) { if (ekat::argv_matches(argv[i], "-b", "--baseline-file")) { @@ -60,10 +60,8 @@ int run_yakl(int argc, char** argv) { inputfile = argv[i]; } // RRTMGP baselines tests to not use kokoks. Swallow the arg, but ignore it - if (std::string(argv[i])=="--ekat-kokkos-device") { - expect_another_arg(i, argc); - ++i; - device = argv[i]; + if (std::string(argv[i])=="--kokkos-device-id=") { + continue; } } @@ -352,10 +350,8 @@ int run_kokkos(int argc, char** argv) { inputfile = argv[i]; } // RRTMGP baselines tests to not use kokoks. Swallow the arg, but ignore it - if (std::string(argv[i])=="--ekat-kokkos-device") { - expect_another_arg(i, argc); - ++i; - device = argv[i]; + if (std::string(argv[i])=="--kokkos-device-id=") { + continue; } } diff --git a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp index 99a582445616..06ca64af5f24 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp @@ -850,6 +850,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop") { #ifdef RRTMGP_ENABLE_KOKKOS using interface_t = scream::rrtmgp::rrtmgp_interface<>; +using pool_t = interface_t::pool_t; using real1dk = interface_t::view_t; using real2dk = interface_t::view_t; using real3dk = interface_t::view_t; @@ -861,6 +862,7 @@ using MDRP = interface_t::MDRP; TEST_CASE("rrtmgp_test_heating_k") { // Initialize Kokkos scream::init_kls(); + pool_t::init(10000); // Test heating rate function by passing simple inputs auto dp = real2dk("dp", 1, 1); @@ -908,12 +910,14 @@ TEST_CASE("rrtmgp_test_heating_k") { REQUIRE(chc(heating)(0,0) == heating_ref); // Clean up + pool_t::finalize(); scream::finalize_kls(); } TEST_CASE("rrtmgp_test_mixing_ratio_to_cloud_mass_k") { // Initialize YAKL scream::init_kls(); + pool_t::init(10000); using physconst = scream::physics::Constants; @@ -966,12 +970,14 @@ TEST_CASE("rrtmgp_test_mixing_ratio_to_cloud_mass_k") { REQUIRE(chc(cloud_mass)(0,0) == cloud_mass_ref); // Clean up + pool_t::finalize(); scream::finalize_kls(); } TEST_CASE("rrtmgp_test_limit_to_bounds_k") { // Initialize YAKL scream::init_kls(); + pool_t::init(10000); // Test limiter function auto arr = real2dk("arr", 2, 2); @@ -998,6 +1004,8 @@ TEST_CASE("rrtmgp_test_limit_to_bounds_k") { REQUIRE(chc(arr_limited)(0,1) == 2.0); REQUIRE(chc(arr_limited)(1,0) == 3.0); REQUIRE(chc(arr_limited)(1,1) == 3.5); + + pool_t::finalize(); scream::finalize_kls(); } @@ -1070,6 +1078,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { // Initialize YAKL scream::init_kls(); + pool_t::init(10000); // Create arrays const int ncol = 1; @@ -1227,6 +1236,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { logger->info("Free memory...\n"); interface_t::rrtmgp_finalize(); gas_concs.reset(); + pool_t::finalize(); scream::finalize_kls(); } @@ -1255,6 +1265,7 @@ TEST_CASE("rrtmgp_test_radiation_do_k") { TEST_CASE("rrtmgp_test_check_range_k") { // Initialize YAKL scream::init_kls(); + pool_t::init(10000); // Create some dummy data and test with both values inside valid range and outside auto dummy = real2dk("dummy", 2, 1); // All values within range @@ -1266,12 +1277,14 @@ TEST_CASE("rrtmgp_test_check_range_k") { // At least one value above upper bound Kokkos::parallel_for(1, KOKKOS_LAMBDA (int i) {dummy(i, 0) = 1.1;}); REQUIRE(scream::rrtmgp::check_range_k(dummy, 0.0, 1.0, "dummy") == false); + pool_t::finalize(); scream::finalize_kls(); } TEST_CASE("rrtmgp_test_subcol_gen_k") { // Initialize YAKL scream::init_kls(); + pool_t::init(10000); // Create dummy data const int ncol = 1; const int nlay = 4; @@ -1291,7 +1304,7 @@ TEST_CASE("rrtmgp_test_subcol_gen_k") { for (unsigned seed = 0; seed < 10; seed++) { auto seeds = int1dk("seeds", ncol); Kokkos::deep_copy(seeds, seed); - cldmask = interface_t::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds); + interface_t::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds, cldmask); // Check answers by computing new cldfrac from mask Kokkos::deep_copy(cldfrac_from_mask, 0.0); Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { @@ -1325,7 +1338,7 @@ TEST_CASE("rrtmgp_test_subcol_gen_k") { for (unsigned seed = 0; seed < 10; seed++) { auto seeds = int1dk("seeds", ncol); Kokkos::deep_copy(seeds, seed); - cldmask = interface_t::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds); + interface_t::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds, cldmask); auto cldmask_h = chc(cldmask); for (int igpt = 0; igpt < ngpt; igpt++) { if (cldmask_h(0,0,igpt) == 1) { @@ -1334,12 +1347,14 @@ TEST_CASE("rrtmgp_test_subcol_gen_k") { } } // Clean up after test + pool_t::finalize(); scream::finalize_kls(); } TEST_CASE("rrtmgp_cloud_area_k") { // Initialize YAKL scream::init_kls(); + pool_t::init(10000); // Create dummy data const int ncol = 1; const int nlay = 2; @@ -1429,12 +1444,14 @@ TEST_CASE("rrtmgp_cloud_area_k") { REQUIRE(chc(cldtot)(0) == 0.0); interface_t::compute_cloud_area(ncol, nlay, ngpt, 100, 300, pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 2.0 / 3.0); + pool_t::finalize(); scream::finalize_kls(); } TEST_CASE("rrtmgp_aerocom_cloudtop_k") { // Initialize YAKL scream::init_kls(); + pool_t::init(10000); // Create dummy data const int ncol = 1; @@ -1623,6 +1640,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { REQUIRE(chc(cldfrac_ice_at_cldtop)(0) == 0.7); // max // cleanup + pool_t::finalize(); scream::finalize_kls(); } #endif diff --git a/components/eamxx/src/physics/share/CMakeLists.txt b/components/eamxx/src/physics/share/CMakeLists.txt index d3c64a4a0b9c..44ce0aba57b8 100644 --- a/components/eamxx/src/physics/share/CMakeLists.txt +++ b/components/eamxx/src/physics/share/CMakeLists.txt @@ -3,11 +3,6 @@ set(PHYSICS_SHARE_SRCS physics_share.cpp physics_test_data.cpp scream_trcmix.cpp - ${SCREAM_BASE_DIR}/../eam/src/physics/cam/physics_utils.F90 - ${SCREAM_BASE_DIR}/../eam/src/physics/cam/scream_abortutils.F90 - ${SCREAM_BASE_DIR}/../eam/src/physics/cam/wv_sat_scream.F90 - ${SCREAM_BASE_DIR}/../eam/src/physics/p3/scream/micro_p3_utils.F90 - ${SCREAM_BASE_DIR}/../eam/src/physics/cam/debug_info.F90 ) # Add ETI source files if not on CUDA/HIP diff --git a/components/eamxx/src/physics/share/physics_constants.hpp b/components/eamxx/src/physics/share/physics_constants.hpp index 9c6e8c339478..c8c638ca88c6 100644 --- a/components/eamxx/src/physics/share/physics_constants.hpp +++ b/components/eamxx/src/physics/share/physics_constants.hpp @@ -122,121 +122,6 @@ struct Constants static constexpr Scalar earth_ellipsoid3 = 1.175; // third expansion coefficient for WGS84 ellipsoid }; -template -struct P3_Constants -{ - public: - Scalar p3_autoconversion_prefactor = 1350.0; - Scalar p3_mu_r_constant = 1.0; - Scalar p3_spa_to_nc = 1.0; - Scalar p3_k_accretion = 67.0; - Scalar p3_eci = 0.5; - Scalar p3_eri = 1.0; - Scalar p3_rho_rime_min = 50.0; - Scalar p3_rho_rime_max = 900.0; - Scalar p3_a_imm = 0.65; - Scalar p3_dep_nucleation_exponent = 0.304; - Scalar p3_ice_sed_knob = 1.0; - Scalar p3_d_breakup_cutoff = 0.00028; - - void set_p3_from_namelist(ekat::ParameterList ¶ms){ - - std::string nname = "p3_autoconversion_prefactor"; - if(params.isParameter(nname)) - p3_autoconversion_prefactor = params.get(nname); - - nname = "p3_mu_r_constant"; - if(params.isParameter(nname)) - p3_mu_r_constant = params.get(nname); - - nname = "p3_spa_to_nc"; - if(params.isParameter(nname)) - p3_spa_to_nc = params.get(nname); - - nname = "p3_k_accretion"; - if(params.isParameter(nname)) - p3_k_accretion = params.get(nname); - - nname = "p3_eci"; - if(params.isParameter(nname)) - p3_eci = params.get(nname); - - nname = "p3_eri"; - if(params.isParameter(nname)) - p3_eri = params.get(nname); - - nname = "p3_rho_rime_min"; - if(params.isParameter(nname)) - p3_rho_rime_min = params.get(nname); - - nname = "p3_rho_rime_max"; - if(params.isParameter(nname)) - p3_rho_rime_max = params.get(nname); - - nname = "p3_a_imm"; - if(params.isParameter(nname)) - p3_a_imm = params.get(nname); - - nname = "p3_dep_nucleation_exponent"; - if(params.isParameter(nname)) - p3_dep_nucleation_exponent = params.get(nname); - - nname = "p3_ice_sed_knob"; - if(params.isParameter(nname)) - p3_ice_sed_knob = params.get(nname); - - nname = "p3_d_breakup_cutoff"; - if(params.isParameter(nname)) - p3_d_breakup_cutoff = params.get(nname); - - }; - - void print_p3constants(std::shared_ptr logger){ - logger->info("P3 Constants:"); - - std::string nname = "p3_autoconversion_prefactor"; - logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_autoconversion_prefactor)); - - nname = "p3_mu_r_constant"; - logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_mu_r_constant)); - - nname = "p3_spa_to_nc"; - logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_spa_to_nc)); - - nname = "p3_k_accretion"; - logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_k_accretion)); - - nname = "p3_eci"; - logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_eci)); - - nname = "p3_eri"; - logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_eri)); - - nname = "p3_rho_rime_min"; - logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_rho_rime_min)); - - nname = "p3_rho_rime_max"; - logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_rho_rime_max)); - - nname = "p3_a_imm"; - logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_a_imm)); - - nname = "p3_dep_nucleation_exponent"; - logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_dep_nucleation_exponent)); - - nname = "p3_ice_sed_knob"; - logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_ice_sed_knob)); - - nname = "p3_d_breakup_cutoff"; - logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_d_breakup_cutoff)); - - logger->info(" "); - }; - - //one can implement a check here too, for acceptable ranges - -}; // P3_Constants - // Gases // Define the molecular weight for each gas, which can then be // used to determine the volume mixing ratio for each gas. diff --git a/components/eamxx/src/physics/share/physics_share_f2c.F90 b/components/eamxx/src/physics/share/physics_share_f2c.F90 index 1bd3a7d21d72..885b026eac74 100644 --- a/components/eamxx/src/physics/share/physics_share_f2c.F90 +++ b/components/eamxx/src/physics/share/physics_share_f2c.F90 @@ -101,7 +101,7 @@ function scream_expm1(input) bind(C) ! return real(kind=c_real) :: scream_expm1 end function scream_expm1 - + function scream_tanh(input) bind(C) use iso_c_binding diff --git a/components/eamxx/src/physics/share/physics_test_data.cpp b/components/eamxx/src/physics/share/physics_test_data.cpp index 4a13528e98b6..f6c6d733fd0e 100644 --- a/components/eamxx/src/physics/share/physics_test_data.cpp +++ b/components/eamxx/src/physics/share/physics_test_data.cpp @@ -26,4 +26,21 @@ PhysicsTestData& PhysicsTestData::assignment_impl(const PhysicsTestData& rhs) return *this; } +void PhysicsTestData::read(const ekat::FILEPtr& fid) +{ + EKAT_REQUIRE_MSG(fid, + "Tried to read from missing file. You may have forgotten to generate baselines for some BFB unit tests"); + m_reals.read(fid); + m_ints.read(fid); + m_bools.read(fid); +} + +void PhysicsTestData::write(const ekat::FILEPtr& fid) const +{ + m_reals.write(fid); + m_ints.write(fid); + m_bools.write(fid); +} + + } // namespace scream diff --git a/components/eamxx/src/physics/share/physics_test_data.hpp b/components/eamxx/src/physics/share/physics_test_data.hpp index 099480b44393..ddd7d77fa860 100644 --- a/components/eamxx/src/physics/share/physics_test_data.hpp +++ b/components/eamxx/src/physics/share/physics_test_data.hpp @@ -5,6 +5,7 @@ #include "ekat/util/ekat_math_utils.hpp" #include "ekat/ekat_assert.hpp" +#include "ekat/util/ekat_file_utils.hpp" #include #include @@ -68,34 +69,81 @@ struct SHOCGridData : public PhysicsTestData { #define PTD_DATA_COPY_CTOR(name, num_args) \ name(const name& rhs) : name(PTD_ONES(num_args)) { *this = rhs; } -#define PTD_ASS0( ) ((void) (0)) -#define PTD_ASS1(a ) a = rhs.a -#define PTD_ASS2(a, b ) PTD_ASS1(a) ; b = rhs.b -#define PTD_ASS3(a, b, c ) PTD_ASS2(a, b) ; c = rhs.c -#define PTD_ASS4(a, b, c, d ) PTD_ASS3(a, b, c) ; d = rhs.d -#define PTD_ASS5(a, b, c, d, e ) PTD_ASS4(a, b, c, d) ; e = rhs.e -#define PTD_ASS6(a, b, c, d, e, f ) PTD_ASS5(a, b, c, d, e) ; f = rhs.f -#define PTD_ASS7(a, b, c, d, e, f, g ) PTD_ASS6(a, b, c, d, e, f) ; g = rhs.g -#define PTD_ASS8(a, b, c, d, e, f, g, h ) PTD_ASS7(a, b, c, d, e, f, g) ; h = rhs.h -#define PTD_ASS9(a, b, c, d, e, f, g, h, i ) PTD_ASS8(a, b, c, d, e, f, g, h) ; i = rhs.i -#define PTD_ASS10(a, b, c, d, e, f, g, h, i, j ) PTD_ASS9(a, b, c, d, e, f, g, h, i) ; j = rhs.j -#define PTD_ASS11(a, b, c, d, e, f, g, h, i, j, k ) PTD_ASS10(a, b, c, d, e, f, g, h, i, j) ; k = rhs.k -#define PTD_ASS12(a, b, c, d, e, f, g, h, i, j, k, l ) PTD_ASS11(a, b, c, d, e, f, g, h, i, j, k) ; l = rhs.l -#define PTD_ASS13(a, b, c, d, e, f, g, h, i, j, k, l, m ) PTD_ASS12(a, b, c, d, e, f, g, h, i, j, k, l) ; m = rhs.m -#define PTD_ASS14(a, b, c, d, e, f, g, h, i, j, k, l, m, n ) PTD_ASS13(a, b, c, d, e, f, g, h, i, j, k, l, m) ; n = rhs.n -#define PTD_ASS15(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o ) PTD_ASS14(a, b, c, d, e, f, g, h, i, j, k, l, m, n) ; o = rhs.o -#define PTD_ASS16(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p ) PTD_ASS15(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o) ; p = rhs.p -#define PTD_ASS17(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q ) PTD_ASS16(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p) ; q = rhs.q -#define PTD_ASS18(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r ) PTD_ASS17(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q) ; r = rhs.r -#define PTD_ASS19(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s ) PTD_ASS18(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r) ; s = rhs.s -#define PTD_ASS20(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t ) PTD_ASS19(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s) ; t = rhs.t +#define PTD_ASS0() ((void) (0)) +#define PTD_ASS1(first) first = rhs.first; PTD_ASS0() +#define PTD_ASS2(first, ...) first = rhs.first; PTD_ASS1(__VA_ARGS__) +#define PTD_ASS3(first, ...) first = rhs.first; PTD_ASS2(__VA_ARGS__) +#define PTD_ASS4(first, ...) first = rhs.first; PTD_ASS3(__VA_ARGS__) +#define PTD_ASS5(first, ...) first = rhs.first; PTD_ASS4(__VA_ARGS__) +#define PTD_ASS6(first, ...) first = rhs.first; PTD_ASS5(__VA_ARGS__) +#define PTD_ASS7(first, ...) first = rhs.first; PTD_ASS6(__VA_ARGS__) +#define PTD_ASS8(first, ...) first = rhs.first; PTD_ASS7(__VA_ARGS__) +#define PTD_ASS9(first, ...) first = rhs.first; PTD_ASS8(__VA_ARGS__) +#define PTD_ASS10(first, ...) first = rhs.first; PTD_ASS9(__VA_ARGS__) +#define PTD_ASS11(first, ...) first = rhs.first; PTD_ASS10(__VA_ARGS__) +#define PTD_ASS12(first, ...) first = rhs.first; PTD_ASS11(__VA_ARGS__) +#define PTD_ASS13(first, ...) first = rhs.first; PTD_ASS12(__VA_ARGS__) +#define PTD_ASS14(first, ...) first = rhs.first; PTD_ASS13(__VA_ARGS__) +#define PTD_ASS15(first, ...) first = rhs.first; PTD_ASS14(__VA_ARGS__) +#define PTD_ASS16(first, ...) first = rhs.first; PTD_ASS15(__VA_ARGS__) +#define PTD_ASS17(first, ...) first = rhs.first; PTD_ASS16(__VA_ARGS__) +#define PTD_ASS18(first, ...) first = rhs.first; PTD_ASS17(__VA_ARGS__) +#define PTD_ASS19(first, ...) first = rhs.first; PTD_ASS18(__VA_ARGS__) +#define PTD_ASS20(first, ...) first = rhs.first; PTD_ASS19(__VA_ARGS__) + +#define PTD_RW0(action) ((void) (0)) +#define PTD_RW1(action, first) ekat::action(&first, 1, fid); PTD_RW0(action) +#define PTD_RW2(action, first, ...) ekat::action(&first, 1, fid); PTD_RW1(action, __VA_ARGS__) +#define PTD_RW3(action, first, ...) ekat::action(&first, 1, fid); PTD_RW2(action, __VA_ARGS__) +#define PTD_RW4(action, first, ...) ekat::action(&first, 1, fid); PTD_RW3(action, __VA_ARGS__) +#define PTD_RW5(action, first, ...) ekat::action(&first, 1, fid); PTD_RW4(action, __VA_ARGS__) +#define PTD_RW6(action, first, ...) ekat::action(&first, 1, fid); PTD_RW5(action, __VA_ARGS__) +#define PTD_RW7(action, first, ...) ekat::action(&first, 1, fid); PTD_RW6(action, __VA_ARGS__) +#define PTD_RW8(action, first, ...) ekat::action(&first, 1, fid); PTD_RW7(action, __VA_ARGS__) +#define PTD_RW9(action, first, ...) ekat::action(&first, 1, fid); PTD_RW8(action, __VA_ARGS__) +#define PTD_RW10(action, first, ...) ekat::action(&first, 1, fid); PTD_RW9(action, __VA_ARGS__) +#define PTD_RW11(action, first, ...) ekat::action(&first, 1, fid); PTD_RW10(action, __VA_ARGS__) +#define PTD_RW12(action, first, ...) ekat::action(&first, 1, fid); PTD_RW11(action, __VA_ARGS__) +#define PTD_RW13(action, first, ...) ekat::action(&first, 1, fid); PTD_RW12(action, __VA_ARGS__) +#define PTD_RW14(action, first, ...) ekat::action(&first, 1, fid); PTD_RW13(action, __VA_ARGS__) +#define PTD_RW15(action, first, ...) ekat::action(&first, 1, fid); PTD_RW14(action, __VA_ARGS__) +#define PTD_RW16(action, first, ...) ekat::action(&first, 1, fid); PTD_RW15(action, __VA_ARGS__) +#define PTD_RW17(action, first, ...) ekat::action(&first, 1, fid); PTD_RW16(action, __VA_ARGS__) +#define PTD_RW18(action, first, ...) ekat::action(&first, 1, fid); PTD_RW17(action, __VA_ARGS__) +#define PTD_RW19(action, first, ...) ekat::action(&first, 1, fid); PTD_RW18(action, __VA_ARGS__) +#define PTD_RW20(action, first, ...) ekat::action(&first, 1, fid); PTD_RW19(action, __VA_ARGS__) +#define PTD_RW21(action, first, ...) ekat::action(&first, 1, fid); PTD_RW20(action, __VA_ARGS__) +#define PTD_RW22(action, first, ...) ekat::action(&first, 1, fid); PTD_RW21(action, __VA_ARGS__) +#define PTD_RW23(action, first, ...) ekat::action(&first, 1, fid); PTD_RW22(action, __VA_ARGS__) +#define PTD_RW24(action, first, ...) ekat::action(&first, 1, fid); PTD_RW23(action, __VA_ARGS__) +#define PTD_RW25(action, first, ...) ekat::action(&first, 1, fid); PTD_RW24(action, __VA_ARGS__) +#define PTD_RW26(action, first, ...) ekat::action(&first, 1, fid); PTD_RW25(action, __VA_ARGS__) +#define PTD_RW27(action, first, ...) ekat::action(&first, 1, fid); PTD_RW26(action, __VA_ARGS__) +#define PTD_RW28(action, first, ...) ekat::action(&first, 1, fid); PTD_RW27(action, __VA_ARGS__) +#define PTD_RW29(action, first, ...) ekat::action(&first, 1, fid); PTD_RW28(action, __VA_ARGS__) +#define PTD_RW30(action, first, ...) ekat::action(&first, 1, fid); PTD_RW29(action, __VA_ARGS__) +#define PTD_RW31(action, first, ...) ekat::action(&first, 1, fid); PTD_RW30(action, __VA_ARGS__) #define PTD_ASSIGN_OP(name, num_scalars, ...) \ name& operator=(const name& rhs) { PTD_ASS##num_scalars(__VA_ARGS__); assignment_impl(rhs); return *this; } +#define PTD_RW_SCALARS(num_scalars, ...) \ + void read_scalars(const ekat::FILEPtr& fid) { EKAT_REQUIRE_MSG(fid, "Tried to read from missing file. You may have forgotten to generate baselines for some BFB unit tests"); PTD_RW##num_scalars(read, __VA_ARGS__); } \ + void write_scalars(const ekat::FILEPtr& fid) const { PTD_RW##num_scalars(write, __VA_ARGS__); } + +#define PTD_RW_SCALARS_ONLY(num_scalars, ...) \ + void read(const ekat::FILEPtr& fid) { EKAT_REQUIRE_MSG(fid, "Tried to read from missing file. You may have forgotten to generate baselines for some BFB unit tests"); PTD_RW##num_scalars(read, __VA_ARGS__); } \ + void write(const ekat::FILEPtr& fid) const { PTD_RW##num_scalars(write, __VA_ARGS__); } + +#define PTD_RW() \ + void read(const ekat::FILEPtr& fid) { read_scalars(fid); PhysicsTestData::read(fid); } \ + void write(const ekat::FILEPtr& fid) const { write_scalars(fid); PhysicsTestData::write(fid); } + #define PTD_STD_DEF(name, num_scalars, ...) \ PTD_DATA_COPY_CTOR(name, num_scalars); \ - PTD_ASSIGN_OP(name, num_scalars, __VA_ARGS__) + PTD_ASSIGN_OP(name, num_scalars, __VA_ARGS__) \ + PTD_RW() \ + PTD_RW_SCALARS(num_scalars, __VA_ARGS__) namespace scream { @@ -241,6 +289,16 @@ class PhysicsTestData m_data = new_data; } + void read(const ekat::FILEPtr& fid) + { + ekat::read(m_data.data(), m_data.size(), fid); + } + + void write(const ekat::FILEPtr& fid) const + { + ekat::write(m_data.data(), m_data.size(), fid); + } + std::vector > m_dims_list; // list of dims, one per unique set of dims std::vector > m_members_list; // list of member pointers, same outer index space as m_dims_list std::vector m_data; // the member data in a flat vector @@ -336,6 +394,10 @@ class PhysicsTestData } } + void read(const ekat::FILEPtr& fid); + + void write(const ekat::FILEPtr& fid) const; + protected: PhysicsTestData& assignment_impl(const PhysicsTestData& rhs); diff --git a/components/eamxx/src/physics/shoc/CMakeLists.txt b/components/eamxx/src/physics/shoc/CMakeLists.txt index 4796b30abefd..3686c3cce7fc 100644 --- a/components/eamxx/src/physics/shoc/CMakeLists.txt +++ b/components/eamxx/src/physics/shoc/CMakeLists.txt @@ -1,19 +1,7 @@ set(SHOC_SRCS - shoc_f90.cpp - shoc_ic_cases.cpp - shoc_iso_c.f90 - shoc_iso_f.f90 - ${SCREAM_BASE_DIR}/../eam/src/physics/cam/shoc.F90 eamxx_shoc_process_interface.cpp ) -if (NOT SCREAM_LIB_ONLY) - list(APPEND SHOC_SRCS - shoc_functions_f90.cpp - shoc_main_wrap.cpp - ) # Add f90 bridges needed for testing -endif() - set(SHOC_HEADERS shoc.hpp eamxx_shoc_process_interface.hpp diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 99e0d83ebf92..1bbb17f84806 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -61,7 +61,7 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids add_field("surf_evap", scalar2d , kg/(m2*s), grid_name); add_field ("T_mid", scalar3d_mid, K, grid_name, ps); - add_field ("qv", scalar3d_mid, kg/kg, grid_name, "tracers", ps); + add_tracer("qv", m_grid, kg/kg, ps); // If TMS is a process, add surface drag coefficient to required fields if (m_params.get("apply_tms", false)) { @@ -75,12 +75,12 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids add_field("phis", scalar2d , m2/s2, grid_name, ps); // Input/Output variables - add_field("tke", scalar3d_mid, m2/s2, grid_name, "tracers", ps); add_field("horiz_winds", vector3d_mid, m/s, grid_name, ps); add_field("sgs_buoy_flux", scalar3d_mid, K*(m/s), grid_name, ps); add_field("eddy_diff_mom", scalar3d_mid, m2/s, grid_name, ps); - add_field("qc", scalar3d_mid, kg/kg, grid_name, "tracers", ps); add_field("cldfrac_liq", scalar3d_mid, nondim, grid_name, ps); + add_tracer("tke", m_grid, m2/s2, ps); + add_tracer("qc", m_grid, kg/kg, ps); // Output variables add_field("pbl_height", scalar2d , m, grid_name); @@ -449,7 +449,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) const auto ncols = m_num_cols; view_1d cell_length("cell_length", ncols); if (m_grid->has_geometry_data("dx_short")) { - // We must be running with IntensiveObservationPeriod on, with a planar geometry + // In this case IOP is running with a planar geometry auto dx = m_grid->get_geometry_data("dx_short").get_view()(); Kokkos::deep_copy(cell_length, dx*1000); // convert km -> m } else { diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp index 4d76efd4558b..de38a7cac7f7 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp @@ -138,8 +138,9 @@ class SHOCMacrophysics : public scream::AtmosphereProcess // Dry static energy shoc_s(i,k) = PF::calculate_dse(T_mid(i,k),z_mid(i,k),phis(i)); + + if (k+1 == nlev_packs) zi_grid(i,nlevi_v)[nlevi_p] = 0; }); - zi_grid(i,nlevi_v)[nlevi_p] = 0; team.team_barrier(); const auto zt_grid_s = ekat::subview(zt_grid, i); diff --git a/components/eamxx/src/physics/shoc/impl/shoc_pblintd_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_pblintd_impl.hpp index 34104e9a8fe5..9c4cef44d1c9 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_pblintd_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_pblintd_impl.hpp @@ -78,7 +78,16 @@ void Functions::pblintd( // Initialize bool check = true; + // The loop below fixes valgrind uninitialized mem errs. As in other + // places in eamxx, we use SCREAM_SHORT_TESTS as a proxy for knowing + // mem checking is on. +#if !defined(NDEBUG) || defined(SCREAM_SHORT_TESTS) + for (size_t i=0; i1 - set (TEST_SUFFIX _omp${SCREAM_TEST_MAX_THREADS}) - endif() - set_tests_properties (shoc_run_and_cmp_f90${TEST_SUFFIX} PROPERTIES LABELS "baseline_gen;baseline_cmp") - set_tests_properties (shoc_run_and_cmp_cxx${TEST_SUFFIX} PROPERTIES LABELS "baseline_gen;cxx baseline_cmp") +# If small kernels are ON, we don't need a separate executable to test them. +# Also, we never want to generate baselines with this separate executable +if (NOT SCREAM_SHOC_SMALL_KERNELS AND NOT SCREAM_ONLY_GENERATE_BASELINES) + CreateUnitTest(shoc_sk_tests "shoc_main_tests.cpp" + LIBS shoc_sk shoc_test_infra + THREADS ${SHOC_THREADS} + EXE_ARGS "--args ${BASELINE_FILE_ARG}" + LABELS "shoc;physics;baseline_cmp" + ) endif() + +CreateUnitTest(shoc_run_and_cmp "shoc_run_and_cmp.cpp" + LIBS shoc shoc_test_infra + EXCLUDE_MAIN_CPP + THREADS ${SCREAM_TEST_MAX_THREADS} + EXE_ARGS "${BASELINE_FILE_ARG}" + LABELS "shoc;physics;baseline_gen;baseline_cmp") diff --git a/components/eamxx/src/physics/shoc/tests/infra/CMakeLists.txt b/components/eamxx/src/physics/shoc/tests/infra/CMakeLists.txt new file mode 100644 index 000000000000..0c0c75ca6aa8 --- /dev/null +++ b/components/eamxx/src/physics/shoc/tests/infra/CMakeLists.txt @@ -0,0 +1,10 @@ +set(INFRA_SRCS + shoc_data.cpp + shoc_ic_cases.cpp + shoc_main_wrap.cpp + shoc_test_data.cpp +) + +add_library(shoc_test_infra ${INFRA_SRCS}) +target_link_libraries(shoc_test_infra shoc) +target_include_directories(shoc_test_infra PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/components/eamxx/src/physics/shoc/shoc_f90.cpp b/components/eamxx/src/physics/shoc/tests/infra/shoc_data.cpp similarity index 87% rename from components/eamxx/src/physics/shoc/shoc_f90.cpp rename to components/eamxx/src/physics/shoc/tests/infra/shoc_data.cpp index bc14a9110d52..553a88f121b6 100644 --- a/components/eamxx/src/physics/shoc/shoc_f90.cpp +++ b/components/eamxx/src/physics/shoc/tests/infra/shoc_data.cpp @@ -1,4 +1,4 @@ -#include "shoc_f90.hpp" +#include "shoc_data.hpp" #include "physics_constants.hpp" #include "shoc_ic_cases.hpp" @@ -6,11 +6,6 @@ using scream::Real; using scream::Int; -extern "C" { - void shoc_init_c(int nlev, Real gravit, Real rair, Real rh2o, Real cpair, - Real zvir, Real latvap, Real latice, Real karman, Real p0); - void shoc_use_cxx_c(bool use_cxx); -} namespace scream { namespace shoc { @@ -112,19 +107,6 @@ FortranDataIterator::getfield (Int i) const { return fields_[i]; } -void shoc_init(Int nlev, bool use_fortran, bool force_reinit) { - static bool is_init = false; - if (!is_init || force_reinit) { - using Scalar = Real; - using C = scream::physics::Constants; - - shoc_init_c((int)nlev, C::gravit, C::Rair, C::RH2O, C::Cpair, C::ZVIR, - C::LatVap, C::LatIce, C::Karman, C::P0); - is_init = true; - } - shoc_use_cxx_c(!use_fortran); -} - int test_FortranData () { Int shcol = 1; Int nlev = 128, num_tracers = 1; @@ -132,11 +114,5 @@ int test_FortranData () { return 0; } -int test_shoc_init (bool use_fortran) { - Int nz = 160; - shoc_init(nz, use_fortran); - return 0; -} - } // namespace shoc } // namespace scream diff --git a/components/eamxx/src/physics/shoc/shoc_f90.hpp b/components/eamxx/src/physics/shoc/tests/infra/shoc_data.hpp similarity index 85% rename from components/eamxx/src/physics/shoc/shoc_f90.hpp rename to components/eamxx/src/physics/shoc/tests/infra/shoc_data.hpp index 22d6ff70e426..453ddbcda85f 100644 --- a/components/eamxx/src/physics/shoc/shoc_f90.hpp +++ b/components/eamxx/src/physics/shoc/tests/infra/shoc_data.hpp @@ -16,9 +16,9 @@ struct FortranData { using KT = KokkosTypes; using Scalar = Real; - using Array1 = typename KT::template lview; - using Array2 = typename KT::template lview; - using Array3 = typename KT::template lview; + using Array1 = typename KT::template view_1d; + using Array2 = typename KT::template view_2d; + using Array3 = typename KT::template view_3d; Int shcol, nlev, nlevi, num_qtracers, nadv; @@ -67,9 +67,6 @@ struct FortranDataIterator { void init(const FortranData::Ptr& d); }; -// Initialize SHOC with the given number of levels. -void shoc_init(Int nlev, bool use_fortran=false, bool force_reinit=false); - // We will likely want to remove these checks in the future, as we're not tied // to the exact implementation or arithmetic in SHOC. For now, these checks are // here to establish that the initial regression-testing code gives results that @@ -77,7 +74,6 @@ void shoc_init(Int nlev, bool use_fortran=false, bool force_reinit=false); Int check_against_python(const FortranData& d); int test_FortranData(); -int test_shoc_init(bool use_fortran); } // namespace shoc } // namespace scream diff --git a/components/eamxx/src/physics/shoc/shoc_ic_cases.cpp b/components/eamxx/src/physics/shoc/tests/infra/shoc_ic_cases.cpp similarity index 97% rename from components/eamxx/src/physics/shoc/shoc_ic_cases.cpp rename to components/eamxx/src/physics/shoc/tests/infra/shoc_ic_cases.cpp index 6af7b1ed7c2c..da651ef05bab 100644 --- a/components/eamxx/src/physics/shoc/shoc_ic_cases.cpp +++ b/components/eamxx/src/physics/shoc/tests/infra/shoc_ic_cases.cpp @@ -16,11 +16,10 @@ namespace { // top and then flips everything, so we do the same. //------------------------------------------------------------------------ -using KT = KokkosTypes; using Scalar = Real; -using Array1 = typename KT::template lview; -using Array2 = typename KT::template lview; -using Array3 = typename KT::template lview; +using Array1 = typename FortranData::Array1; +using Array2 = typename FortranData::Array2; +using Array3 = typename FortranData::Array3; // Flip all vertical data in the given array. void flip_vertically(Array2& array) diff --git a/components/eamxx/src/physics/shoc/shoc_ic_cases.hpp b/components/eamxx/src/physics/shoc/tests/infra/shoc_ic_cases.hpp similarity index 93% rename from components/eamxx/src/physics/shoc/shoc_ic_cases.hpp rename to components/eamxx/src/physics/shoc/tests/infra/shoc_ic_cases.hpp index 42c71c2536eb..26c06a3cf125 100644 --- a/components/eamxx/src/physics/shoc/shoc_ic_cases.hpp +++ b/components/eamxx/src/physics/shoc/tests/infra/shoc_ic_cases.hpp @@ -1,7 +1,7 @@ #ifndef INCLUDE_SCREAM_SHOC_IC_CASES_HPP #define INCLUDE_SCREAM_SHOC_IC_CASES_HPP -#include "shoc_f90.hpp" +#include "shoc_data.hpp" namespace scream { namespace shoc { diff --git a/components/eamxx/src/physics/shoc/shoc_main_wrap.cpp b/components/eamxx/src/physics/shoc/tests/infra/shoc_main_wrap.cpp similarity index 66% rename from components/eamxx/src/physics/shoc/shoc_main_wrap.cpp rename to components/eamxx/src/physics/shoc/tests/infra/shoc_main_wrap.cpp index 5edfdcd6d282..0266b1e6b4bb 100644 --- a/components/eamxx/src/physics/shoc/shoc_main_wrap.cpp +++ b/components/eamxx/src/physics/shoc/tests/infra/shoc_main_wrap.cpp @@ -1,6 +1,6 @@ #include "shoc_main_wrap.hpp" -#include "shoc_f90.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_data.hpp" +#include "shoc_test_data.hpp" #include "physics_constants.hpp" #include "shoc_ic_cases.hpp" @@ -8,72 +8,36 @@ using scream::Real; using scream::Int; -extern "C" { - Int shoc_main_c(int shcol, int nlev, int nlevi, Real dtime, int nadv, - Real* host_dx, Real* host_dy, Real* thv, Real* zt_grid, - Real* zi_grid, Real* pres, Real* presi, Real* pdel, - Real* wthl_sfc, Real* wqw_sfc, Real* uw_sfc, Real* vw_sfc, - Real* wtracer_sfc, int num_qtracers, Real* w_field, - Real* inv_exner, Real* phis, Real* host_dse, Real* tke, - Real* thetal, Real* qw, Real* u_wind, Real* v_wind, - Real* qtracers, Real* wthv_sec, Real* tkh, Real* tk, - Real* shoc_ql, Real* shoc_cldfrac, Real* pblh, - Real* shoc_mix, Real* isotropy, Real* w_sec, Real* thl_sec, - Real* qw_sec, Real* qwthl_sec, Real* wthl_sec, Real* wqw_sec, - Real* wtke_sec, Real* uw_sec, Real* vw_sec, Real* w3, - Real* wqls_sec, Real* brunt, Real* shoc_ql2, Real* elapsed_s); -} namespace scream { namespace shoc { -Int shoc_main(FortranData& d, bool use_fortran) { +Int shoc_main(FortranData& d) { EKAT_REQUIRE_MSG(d.dtime > 0, "Invalid dtime"); EKAT_REQUIRE_MSG(d.nadv > 0, "Invalid nadv"); - if (use_fortran) { - Real elapsed_s; - shoc_main_c((int)d.shcol, (int)d.nlev, (int)d.nlevi, d.dtime, (int)d.nadv, - d.host_dx.data(), d.host_dy.data(), d.thv.data(), - d.zt_grid.data(), d.zi_grid.data(), d.pres.data(), d.presi.data(), - d.pdel.data(), d.wthl_sfc.data(), d.wqw_sfc.data(), d.uw_sfc.data(), - d.vw_sfc.data(), d.wtracer_sfc.data(), (int)d.num_qtracers, - d.w_field.data(), d.inv_exner.data(), d.phis.data(), d.host_dse.data(), - d.tke.data(), d.thetal.data(), d.qw.data(), d.u_wind.data(), - d.v_wind.data(), d.qtracers.data(), d.wthv_sec.data(), d.tkh.data(), - d.tk.data(), d.shoc_ql.data(), d.shoc_cldfrac.data(), d.pblh.data(), - d.shoc_mix.data(), d.isotropy.data(), d.w_sec.data(), - d.thl_sec.data(), d.qw_sec.data(), d.qwthl_sec.data(), - d.wthl_sec.data(), d.wqw_sec.data(), d.wtke_sec.data(), - d.uw_sec.data(), d.vw_sec.data(), d.w3.data(), d.wqls_sec.data(), - d.brunt.data(), d.shoc_ql2.data(), &elapsed_s); - return static_cast(elapsed_s * 1000000); - } else { - const int npbl = d.nlev; - return shoc_main_f((int)d.shcol, (int)d.nlev, (int)d.nlevi, d.dtime, (int)d.nadv, - npbl, d.host_dx.data(), d.host_dy.data(), - d.thv.data(), d.zt_grid.data(), d.zi_grid.data(), d.pres.data(), - d.presi.data(), d.pdel.data(), d.wthl_sfc.data(), - d.wqw_sfc.data(), d.uw_sfc.data(), d.vw_sfc.data(), - d.wtracer_sfc.data(), (int)d.num_qtracers, - d.w_field.data(), d.inv_exner.data(), d.phis.data(), d.host_dse.data(), - d.tke.data(), d.thetal.data(), d.qw.data(), - d.u_wind.data(), d.v_wind.data(), d.qtracers.data(), d.wthv_sec.data(), - d.tkh.data(), d.tk.data(), d.shoc_ql.data(), - d.shoc_cldfrac.data(), d.pblh.data(), d.shoc_mix.data(), d.isotropy.data(), - d.w_sec.data(), d.thl_sec.data(), - d.qw_sec.data(), d.qwthl_sec.data(), d.wthl_sec.data(), d.wqw_sec.data(), - d.wtke_sec.data(), d.uw_sec.data(), - d.vw_sec.data(), d.w3.data(), d.wqls_sec.data(), d.brunt.data(), - d.shoc_ql2.data()); - } + const int npbl = d.nlev; + return shoc_main_host((int)d.shcol, (int)d.nlev, (int)d.nlevi, d.dtime, (int)d.nadv, + npbl, d.host_dx.data(), d.host_dy.data(), + d.thv.data(), d.zt_grid.data(), d.zi_grid.data(), d.pres.data(), + d.presi.data(), d.pdel.data(), d.wthl_sfc.data(), + d.wqw_sfc.data(), d.uw_sfc.data(), d.vw_sfc.data(), + d.wtracer_sfc.data(), (int)d.num_qtracers, + d.w_field.data(), d.inv_exner.data(), d.phis.data(), d.host_dse.data(), + d.tke.data(), d.thetal.data(), d.qw.data(), + d.u_wind.data(), d.v_wind.data(), d.qtracers.data(), d.wthv_sec.data(), + d.tkh.data(), d.tk.data(), d.shoc_ql.data(), + d.shoc_cldfrac.data(), d.pblh.data(), d.shoc_mix.data(), d.isotropy.data(), + d.w_sec.data(), d.thl_sec.data(), + d.qw_sec.data(), d.qwthl_sec.data(), d.wthl_sec.data(), d.wqw_sec.data(), + d.wtke_sec.data(), d.uw_sec.data(), + d.vw_sec.data(), d.w3.data(), d.wqls_sec.data(), d.brunt.data(), + d.shoc_ql2.data()); } namespace { -using KT = KokkosTypes; -using Scalar = Real; -using Array2 = typename KT::template lview; -using Array3 = typename KT::template lview; +using Array2 = typename FortranData::Array2; +using Array3 = typename FortranData::Array3; // Returns a string representation of the given 2D array. std::string array_as_string(const Array2& array) @@ -252,9 +216,8 @@ void gen_plot_script(const std::vector >& data, } // end anonymous namespace -int test_shoc_ic (bool use_fortran, bool gen_plot_scripts) { +int test_shoc_ic (bool gen_plot_scripts) { Int nz = 160; - shoc_init(nz, use_fortran); // Here we: // 1. Initialize a standard case with settings identical to // scream-doc/ѕhoc_port/shocintr.py's example_run_case method @@ -272,7 +235,7 @@ int test_shoc_ic (bool use_fortran, bool gen_plot_scripts) { // 3. Run 100 steps, each of size dtime = 10 (as in that method) d->nadv = 100; d->dtime = 10; - shoc_main(*d,use_fortran); + shoc_main(*d); // 4. Generate a Python script that plots the results. { diff --git a/components/eamxx/src/physics/shoc/shoc_main_wrap.hpp b/components/eamxx/src/physics/shoc/tests/infra/shoc_main_wrap.hpp similarity index 80% rename from components/eamxx/src/physics/shoc/shoc_main_wrap.hpp rename to components/eamxx/src/physics/shoc/tests/infra/shoc_main_wrap.hpp index 7ecfa1221a33..f11cf14992d7 100644 --- a/components/eamxx/src/physics/shoc/shoc_main_wrap.hpp +++ b/components/eamxx/src/physics/shoc/tests/infra/shoc_main_wrap.hpp @@ -12,13 +12,12 @@ namespace shoc { struct FortranData; // Run SHOC subroutines, populating inout and out fields of d. -ekat::Int shoc_main(FortranData& d, bool use_fortran); - +ekat::Int shoc_main(FortranData& d); // Test SHOC by running initial conditions for a number of steps and comparing // against reference data. If gen_plot_scripts is true, Python scripts are // emitted that plot initial and final conditions. -int test_shoc_ic(bool use_fortran, bool gen_plot_scripts = false); +int test_shoc_ic(bool gen_plot_scripts = false); } // namespace shoc } // namespace scream diff --git a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp b/components/eamxx/src/physics/shoc/tests/infra/shoc_test_data.cpp similarity index 70% rename from components/eamxx/src/physics/shoc/shoc_functions_f90.cpp rename to components/eamxx/src/physics/shoc/tests/infra/shoc_test_data.cpp index e7468062d3c5..2773c7ad220b 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp +++ b/components/eamxx/src/physics/shoc/tests/infra/shoc_test_data.cpp @@ -1,6 +1,6 @@ -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" -#include "shoc_f90.hpp" +#include "shoc_data.hpp" #include "ekat/ekat_assert.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" @@ -14,834 +14,305 @@ using scream::Real; using scream::Int; -// -// A C interface to SHOC fortran calls. The stubs below will link to fortran definitions in shoc_iso_c.f90 -// - -extern "C" { - -// Special shoc_init function for shoc_main_bfb test -void shoc_init_for_main_bfb_c(int nlev, Real gravit, Real rair, Real rh2o, Real cpair, - Real zvir, Real latvap, Real latice, Real karman, Real p0, - Real* pref_mid, int nbot_shoc, int ntop_shoc); -void shoc_use_cxx_c(bool use_cxx); - - -void shoc_grid_c(int shcol, int nlev, int nlevi, Real *zt_grid, Real *zi_grid, - Real *pdel, Real *dz_zt, Real *dzi_zi, Real *rho_zt); - -void shoc_diag_obklen_c(Int shcol, Real *uw_sfc, Real *vw_sfc, Real *wthl_sfc, - Real *wqw_sfc, Real *thl_sfc, Real *cldliq_sfc, - Real *qv_sfc, Real *ustar, Real *kbfs, Real *obklen); - -void update_host_dse_c(Int shcol, Int nlev, Real *thlm, Real *shoc_ql, - Real *inv_exner, Real *zt_grid, Real *phis, Real *host_dse); - -void shoc_energy_fixer_c(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, - Real *zt_grid, Real *zi_grid, Real *se_b, Real *ke_b, - Real *wv_b, Real *wl_b, Real *se_a, Real *ke_a, - Real *wv_a, Real *wl_a, Real *wthl_sfc, Real *wqw_sfc, - Real *rho_zt, Real *tke, Real *pint, - Real *host_dse); - -void shoc_energy_integrals_c(Int shcol, Int nlev, Real *host_dse, Real *pdel, - Real *rtm, Real *rcm, Real *u_wind, Real *v_wind, - Real *se_int, Real *ke_int, Real *wv_int, Real *wl_int); - -void shoc_energy_total_fixer_c(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, - Real *zt_grid, Real *zi_grid, - Real *se_b, Real *ke_b, Real *wv_b, Real *wl_b, - Real *se_a, Real *ke_a, Real *wv_a, Real *wl_a, - Real *wthl_sfc, Real *wqw_sfc, Real *rho_zt, Real *pint, - Real *te_a, Real *te_b); - -void shoc_energy_threshold_fixer_c(Int shcol, Int nlev, Int nlevi, - Real *pint, Real *tke, Real *te_a, Real *te_b, - Real *se_dis, Int *shoctop); - -void shoc_energy_dse_fixer_c(Int shcol, Int nlev, - Real *se_dis, Int *shoctop, - Real *host_dse); - -void calc_shoc_varorcovar_c(Int shcol, Int nlev, Int nlevi, Real tunefac, - Real *isotropy_zi, Real *tkh_zi, Real *dz_zi, - Real *invar1, Real *invar2, Real *varorcovar); - -void compute_tmpi_c(Int nlevi, Int shcol, Real dtime, Real *rho_zi, - Real *dz_zi, Real *tmpi); - -void dp_inverse_c(Int nlev, Int shcol, Real *rho_zt, Real *dz_zt, Real *rdp_zt); - -void sfc_fluxes_c(Int shcol, Int num_tracer, Real dtime, Real *rho_zi_sfc, - Real *rdp_zt_sfc, Real *wthl_sfc, Real *wqw_sfc, Real *wtracer_sfc, - Real *wtke_sfc, Real *thetal, Real *qw, Real *tke, Real *tracer); - -void impli_srf_stress_term_c(Int shcol, Real *rho_zi_sfc, Real *uw_sfc, - Real *vw_sfc, Real *u_wind_sfc, Real *v_wind_sfc, - Real *ksrf); - -void tke_srf_flux_term_c(Int shcol, Real *uw_sfc, Real *vw_sfc, - Real *wtke_sfc); - -void check_tke_c(Int shcol, Int nlev, Real *tke); - -void shoc_tke_c(Int shcol, Int nlev, Int nlevi, Real dtime, Real *wthv_sec, - Real *shoc_mix, Real *dz_zi, Real *dz_zt, Real *pres, - Real* tabs, Real *u_wind, Real *v_wind, Real *brunt, - Real *zt_grid, Real *zi_grid, Real *pblh, Real *tke, - Real *tk, Real *tkh, Real *isotropy); - -void integ_column_stability_c(Int nlev, Int shcol, Real *dz_zt, Real *pres, - Real *brunt, Real *brunt_int); - -void compute_shr_prod_c(Int nlevi, Int nlev, Int shcol, Real *dz_zi, - Real *u_wind, Real *v_wind, Real *sterm); - -void isotropic_ts_c(Int nlev, Int shcol, Real *brunt_int, Real *tke, - Real *a_diss, Real *brunt, Real *isotropy); - -void adv_sgs_tke_c(Int nlev, Int shcol, Real dtime, Real *shoc_mix, - Real *wthv_sec, Real *sterm_zt, Real *tk, - Real *tke, Real *a_diss); - -void eddy_diffusivities_c(Int nlev, Int shcol, Real *pblh, - Real *zt_grid, Real *tabs, Real *shoc_mix, Real *sterm_zt, - Real *isotropy, Real *tke, Real *tkh, Real *tk); - -void calc_shoc_vertflux_c(Int shcol, Int nlev, Int nlevi, Real *tkh_zi, - Real *dz_zi, Real *invar, Real *vertflux); - -void shoc_length_c(Int shcol, Int nlev, Int nlevi, Real *host_dx, - Real *host_dy, Real *zt_grid, - Real *zi_grid, Real *dz_zt, Real *tke, - Real *thv, Real *brunt, Real *shoc_mix); - -void compute_brunt_shoc_length_c(Int nlev, Int nlevi, Int shcol ,Real *dz_zt, - Real *thv, Real *thv_zi, Real *brunt); - -void compute_l_inf_shoc_length_c(Int nlev, Int shcol, Real *zt_grid, Real *dz_zt, - Real *tke, Real *l_inf); - -void compute_shoc_mix_shoc_length_c(Int nlev, Int shcol, Real *tke, Real* brunt, - Real *zt_grid, Real *l_inf, Real *shoc_mix); - -void check_length_scale_shoc_length_c(Int nlev, Int shcol, Real *host_dx, - Real *host_dy, Real *shoc_mix); - -void clipping_diag_third_shoc_moments_c(Int nlevi, Int shcol, Real *w_sec_zi, - Real *w3); - -void fterms_input_for_diag_third_shoc_moment_c(Real dz_zi, Real dz_zt, Real dz_zt_kc, - Real isotropy_zi, Real brunt_zi, Real thetal_zi, - Real *thedz, Real *thedz2, Real *iso, - Real *isosqrd, Real *buoy_sgs2, Real *bet2); -void f0_to_f5_diag_third_shoc_moment_c(Real thedz, Real thedz2, Real bet2, Real iso, - Real isosqrd, Real wthl_sec, Real wthl_sec_kc, - Real wthl_sec_kb, Real thl_sec_kc, - Real thl_sec_kb, Real w_sec, Real w_sec_kc, Real w_sec_zi, - Real tke, Real tke_kc, Real *f0, Real *f1, - Real *f2, Real *f3, Real *f4, Real *f5); - -void omega_terms_diag_third_shoc_moment_c(Real buoy_sgs2, Real f3, Real f4, - Real *omega0, Real *omega1, Real *omega2); - -void x_y_terms_diag_third_shoc_moment_c(Real buoy_sgs2, Real f0, Real f1, Real f2, - Real *x0, Real *y0, Real *x1, Real *y1); - -void aa_terms_diag_third_shoc_moment_c(Real omega0, Real omega1, Real omega2, - Real x0, Real x1, Real y0, Real y1, - Real *aa0, Real *aa1); - -void w3_diag_third_shoc_moment_c(Real aa0, Real aa1, Real x0, - Real x1, Real f5, Real *w3); -void shoc_diag_second_moments_srf_c(Int shcol, Real* wthl_sfc, Real* uw_sfc, Real* vw_sfc, - Real* ustar2, Real* wstar); - -void diag_third_shoc_moments_c(Int shoc, Int nlev, Int nlevi, Real *w_sec, - Real *thl_sec, - Real *wthl_sec, Real *isotropy, Real *brunt, - Real *thetal, Real *tke, - Real *dz_zt, Real *dz_zi, Real *zt_grid, - Real *zi_grid, Real *w3); - -void compute_diag_third_shoc_moment_c(Int shcol, Int nlev, Int nlevi, Real *w_sec, - Real *thl_sec, Real *wthl_sec, Real *tke, - Real *dz_zt, Real *dz_zi, Real *isotropy_zi, - Real *brunt_zi, Real *w_sec_zi, Real *thetal_zi, - Real *w3); - -void linear_interp_c(Real* x1, Real* x2, Real* y1, Real* y2, Int km1, Int km2, Int ncol, Real minthresh); - -void shoc_assumed_pdf_c(Int shcol, Int nlev, Int nlevi, Real *thetal, Real *qw, - Real *w_first, Real *thl_sec, Real *qw_sec, Real *wthl_sec, - Real *w_sec, Real *wqw_sec, Real *qwthl_sec, Real *w3, - Real *pres, Real *zt_grid, Real *zi_grid, - Real *shoc_cldfrac, Real *shoc_ql, Real *wqls, - Real *wthv_sec, Real *shoc_ql2); - -void shoc_assumed_pdf_tilde_to_real_c(Real w_first, Real sqrtw2, Real* w1); - -void shoc_assumed_pdf_vv_parameters_c(Real w_first, Real w_sec, Real w3var, - Real *Skew_w, Real *w1_1, Real *w1_2, - Real *w2_1, Real *w2_2, Real *a); - -void shoc_assumed_pdf_thl_parameters_c(Real wthlsec, Real sqrtw2, Real sqrtthl, - Real thlsec, Real thl_first, Real w1_1, - Real w1_2, Real Skew_w, Real a, bool dothetal_skew, - Real *thl1_1, Real *thl1_2, Real *thl2_1, - Real *thl2_2, Real *sqrtthl2_1, - Real *sqrtthl2_2); - -void shoc_assumed_pdf_qw_parameters_c(Real wqwsec, Real sqrtw2, Real Skew_w, - Real sqrtqt, Real qw_sec, Real w1_1, - Real w1_2, Real qw_first, Real a, - Real *qw1_1, Real *qw1_2, Real *qw2_1, - Real *qw2_2, Real *sqrtqw2_1, - Real *sqrtqw2_2); - -void shoc_assumed_pdf_inplume_correlations_c(Real sqrtqw2_1, Real sqrtthl2_1, - Real a, Real sqrtqw2_2, Real sqrtthl2_2, - Real qwthlsec, Real qw1_1, Real qw_first, - Real thl1_1, Real thl_first, Real qw1_2, - Real thl1_2, Real *r_qwthl_1); - -void shoc_assumed_pdf_compute_temperature_c(Real thl1, Real basepres, - Real pval, Real *Tl1); - -void shoc_assumed_pdf_compute_qs_c(Real Tl1_1, Real Tl1_2, Real pval, - Real *qs1, Real *beta1, Real *qs2, Real *beta2); - -void shoc_assumed_pdf_compute_s_c(Real qw1, Real qs1, Real beta, Real pval, Real thl2, - Real qw2,Real sqrtthl2, Real sqrtqw2, Real r_qwthl, - Real *s, Real *std_s, Real *qn, Real *C); - -void shoc_assumed_pdf_compute_sgs_liquid_c(Real a, Real ql1, Real ql2, Real *shoc_ql); - -void shoc_assumed_pdf_compute_cloud_liquid_variance_c(Real a, Real s1, Real ql1, - Real C1, Real std_s1, Real s2, Real ql2, Real C2, - Real std_s2, Real shoc_ql, Real *shoc_ql2); - -void shoc_assumed_pdf_compute_liquid_water_flux_c(Real a, Real w1_1, Real w_first, - Real ql1, Real w1_2, Real ql2, Real *wqls); - -void shoc_assumed_pdf_compute_buoyancy_flux_c(Real wthlsec, Real epsterm, Real wqwsec, - Real pval, Real wqls, Real *wthv_sec); - -void shoc_diag_second_moments_ubycond_c(Int shcol, Real* thl_sec, Real* qw_sec, - Real* wthl_sec, Real* wqw_sec, Real* qwthl_sec, - Real* uw_sec, Real* vw_sec, Real* wtke_sec); - -void shoc_pblintd_init_pot_c(Int shcol, Int nlev, Real* thl, Real* ql, Real* q, Real* thv); - -void diag_second_moments_lbycond_c(Int shcol, Real *wthl_sfc, Real *wqw_sfc, Real *uw_sfc, - Real *vw_sfc, Real *ustar2, Real *wstar, Real *wthl_sec, - Real *wqw_sec, Real *uw_sec, Real *vw_sec, Real *wtke_sec, - Real *thl_sec, Real *qw_sec, Real *qwthl_sec); - -void diag_second_moments_c(Int shcol, Int nlev, Int nlevi, Real *thetal, Real *qw, - Real *u_wind, Real *v_wind, Real *tke, Real *isotropy, - Real *tkh, Real *tk, Real *dz_zi, Real *zt_grid, Real *zi_grid, - Real *shoc_mix, Real *thl_sec, Real *qw_sec, Real *wthl_sec, - Real *wqw_sec, Real *qwthl_sec, Real *uw_sec, Real *vw_sec, - Real *wtke_sec, Real *w_sec); - -void diag_second_shoc_moments_c(Int shcol, Int nlev, Int nlevi, Real *thetal, - Real *qw, Real *u_wind, Real *v_wind, Real *tke, - Real *isotropy, Real *tkh, Real *tk, Real *dz_zi, - Real *zt_grid, Real *zi_grid, Real *shoc_mix, - Real *wthl_sfc, Real *wqw_sfc, Real *uw_sfc, - Real *vw_sfc, Real *thl_sec, Real *qw_sec, - Real *wthl_sec, Real *wqw_sec, Real *qwthl_sec, - Real *uw_sec, Real *vw_sec, Real *wtke_sec, Real *w_sec); - -void shoc_pblintd_cldcheck_c(Int shcol, Int nlev, Int nlevi, Real* zi, Real* cldn, Real* pblh); - -void compute_shoc_vapor_c(Int shcol, Int nlev, Real* qw, Real* ql, Real* qv); - -void update_prognostics_implicit_c(Int shcol, Int nlev, Int nlevi, Int num_tracer, Real dtime, - Real* dz_zt, Real* dz_zi, Real* rho_zt, Real* zt_grid, Real* zi_grid, - Real* tk, Real* tkh, Real* uw_sfc, Real* vw_sfc, Real* wthl_sfc, - Real* wqw_sfc, Real* wtracer_sfc, Real* thetal, Real* qw, Real* tracer, - Real* tke, Real* u_wind, Real* v_wind); - -void shoc_main_c(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Real* host_dx, Real* host_dy, - Real* thv, Real* zt_grid, Real* zi_grid, Real* pres, Real* presi, Real* pdel, - Real* wthl_sfc, Real* wqw_sfc, Real* uw_sfc, Real* vw_sfc, Real* wtracer_sfc, - Int num_qtracers, Real* w_field, Real* inv_exner, Real* phis, Real* host_dse, Real* tke, - Real* thetal, Real* qw, Real* u_wind, Real* v_wind, Real* qtracers, Real* wthv_sec, - Real* tkh, Real* tk, Real* shoc_ql, Real* shoc_cldfrac, Real* pblh, Real* shoc_mix, - Real* isotropy, Real* w_sec, Real* thl_sec, Real* qw_sec, Real* qwthl_sec, Real* wthl_sec, - Real* wqw_sec, Real* wtke_sec, Real* uw_sec, Real* vw_sec, Real* w3, Real* wqls_sec, - Real* brunt, Real* shoc_ql2, Real* elapsed_s); - -void pblintd_height_c(Int shcol, Int nlev, Int npbl_in, Real* z, Real* u, Real* v, Real* ustar, Real* thv, Real* thv_ref, Real* pblh, Real* rino, bool* check); - -void vd_shoc_decomp_c(Int shcol, Int nlev, Int nlevi, Real* kv_term, Real* tmpi, Real* rdp_zt, Real dtime, - Real* flux, Real* du, Real* dl, Real* d); - -void vd_shoc_solve_c(Int shcol, Int nlev, Real* du, Real* dl, Real* d, Real* var); -void pblintd_surf_temp_c(Int shcol, Int nlev, Int nlevi, Real* z, Real* ustar, Real* obklen, Real* kbfs, Real* thv, Real* tlv, Real* pblh, bool* check, Real* rino); -void pblintd_check_pblh_c(Int shcol, Int nlev, Int nlevi, Real* z, Real* ustar, bool* check, Real* pblh); -void pblintd_c(Int shcol, Int nlev, Int nlevi, Int npbl_in, Real* z, Real* zi, Real* thl, Real* ql, Real* q, Real* u, Real* v, Real* ustar, Real* obklen, Real* kbfs, Real* cldn, Real* pblh); - -void compute_shoc_temperature_c(Int shcol, Int nlev, Real* thetal, Real*ql, Real* inv_exner, Real* tabs); - -} // extern "C" : end _c decls - namespace scream { namespace shoc { // -// Glue functions to call fortran from from C++ with the Data struct +// Glue functions to call from host with the Data struct // -// In all C++ -> Fortran bridge functions you should see shoc_init(nlev, true). -// We are provisionally following P3 here in case SHOC uses global data. The -// 'true' argument is to set shoc to use its fortran implementations instead of -// calling back to C++. We want this behavior since it doesn't make much sense -// for C++ to bridge over to fortran only to have fortran bridge back to C++. -// Anyone who wants the C++ implementation should call it directly. We need -// need to be aware of data layout since f90 is different from cxx. All these -// functions will expect incoming data to be C layout. They will transpose to f90 -// before calling fortran and then back to C before returning. +// We are provisionally following P3 here in case SHOC uses global data. // void shoc_grid(ShocGridData& d) { - shoc_init(d.nlev, true); - d.transpose(); - shoc_grid_c(d.shcol, d.nlev, d.nlevi, d.zt_grid, d.zi_grid, d.pdel, d.dz_zt, d.dz_zi, d.rho_zt); - d.transpose(); + shoc_grid_host(d.shcol, d.nlev, d.nlevi, d.zt_grid, d.zi_grid, d.pdel, d.dz_zt, d.dz_zi, d.rho_zt); } void shoc_diag_obklen(ShocDiagObklenData& d) { - shoc_init(1, true); // single level function - shoc_diag_obklen_c(d.shcol, d.uw_sfc, d.vw_sfc, d.wthl_sfc, d.wqw_sfc, d.thl_sfc, d.cldliq_sfc, d.qv_sfc, d.ustar, d.kbfs, d.obklen); + shoc_diag_obklen_host(d.shcol, d.uw_sfc, d.vw_sfc, d.wthl_sfc, d.wqw_sfc, d.thl_sfc, d.cldliq_sfc, d.qv_sfc, d.ustar, d.kbfs, d.obklen); } void update_host_dse(UpdateHostDseData& d) { - shoc_init(d.nlev, true); - d.transpose(); - update_host_dse_c(d.shcol, d.nlev, d.thlm, d.shoc_ql, d.inv_exner, d.zt_grid, d.phis, d.host_dse); - d.transpose(); + update_host_dse_host(d.shcol, d.nlev, d.thlm, d.shoc_ql, d.inv_exner, d.zt_grid, d.phis, d.host_dse); } void shoc_energy_fixer(ShocEnergyFixerData& d) { - shoc_init(d.nlev, true); - d.transpose(); - shoc_energy_fixer_c(d.shcol, d.nlev, d.nlevi, d.dtime, d.nadv, d.zt_grid, d.zi_grid, d.se_b, d.ke_b, d.wv_b, d.wl_b, d.se_a, d.ke_a, d.wv_a, d.wl_a, d.wthl_sfc, d.wqw_sfc, d.rho_zt, d.tke, d.pint, d.host_dse); - d.transpose(); + shoc_energy_fixer_host(d.shcol, d.nlev, d.nlevi, d.dtime, d.nadv, d.zt_grid, d.zi_grid, d.se_b, d.ke_b, d.wv_b, d.wl_b, d.se_a, d.ke_a, d.wv_a, d.wl_a, d.wthl_sfc, d.wqw_sfc, d.rho_zt, d.tke, d.pint, d.host_dse); } void shoc_energy_integrals(ShocEnergyIntegralsData& d) { - shoc_init(d.nlev, true); - d.transpose(); - shoc_energy_integrals_c(d.shcol, d.nlev, d.host_dse, d.pdel, d.rtm, d.rcm, d.u_wind, d.v_wind, d.se_int, d.ke_int, d.wv_int, d.wl_int); - d.transpose(); -} - -void shoc_energy_total_fixer(ShocEnergyTotalFixerData& d) -{ - shoc_init(d.nlev, true); - d.transpose(); - shoc_energy_total_fixer_c(d.shcol, d.nlev, d.nlevi, d.dtime, d.nadv, - d.zt_grid, d.zi_grid, d.se_b, d.ke_b, d.wv_b, - d.wl_b, d.se_a, d.ke_a, d.wv_a, d.wl_a, d.wthl_sfc, - d.wqw_sfc, d.rho_zt, d.pint, d.te_a, d.te_b); - d.transpose(); -} - -void shoc_energy_threshold_fixer(ShocEnergyThresholdFixerData& d) -{ - shoc_init(d.nlev, true); - d.transpose(); - shoc_energy_threshold_fixer_c(d.shcol, d.nlev, d.nlevi, d.pint, d.tke, d.te_a, d.te_b, d.se_dis, d.shoctop); - d.transpose(); -} - -void shoc_energy_dse_fixer(ShocEnergyDseFixerData& d) -{ - shoc_init(d.nlev, true); - d.transpose(); - shoc_energy_dse_fixer_c(d.shcol, d.nlev, d.se_dis, d.shoctop, d.host_dse); - d.transpose(); + shoc_energy_integrals_host(d.shcol, d.nlev, d.host_dse, d.pdel, d.rtm, d.rcm, d.u_wind, d.v_wind, d.se_int, d.ke_int, d.wv_int, d.wl_int); } void calc_shoc_vertflux(CalcShocVertfluxData& d) { - shoc_init(d.nlev, true); - d.transpose(); - calc_shoc_vertflux_c(d.shcol, d.nlev, d.nlevi, d.tkh_zi, d.dz_zi, d.invar, d.vertflux); - d.transpose(); + calc_shoc_vertflux_host(d.shcol, d.nlev, d.nlevi, d.tkh_zi, d.dz_zi, d.invar, d.vertflux); } void calc_shoc_varorcovar(CalcShocVarorcovarData& d) { - shoc_init(d.nlev, true); - d.transpose(); - calc_shoc_varorcovar_c(d.shcol, d.nlev, d.nlevi, d.tunefac, d.isotropy_zi, d.tkh_zi, d.dz_zi, d.invar1, d.invar2, d.varorcovar); - d.transpose(); + calc_shoc_varorcovar_host(d.shcol, d.nlev, d.nlevi, d.tunefac, d.isotropy_zi, d.tkh_zi, d.dz_zi, d.invar1, d.invar2, d.varorcovar); } void compute_tmpi(ComputeTmpiData& d) { - shoc_init(d.nlevi - 1, true); // nlev = nlevi - 1 - d.transpose(); - compute_tmpi_c(d.nlevi, d.shcol, d.dtime, d.rho_zi, d.dz_zi, d.tmpi); - d.transpose(); + compute_tmpi_host(d.nlevi, d.shcol, d.dtime, d.rho_zi, d.dz_zi, d.tmpi); } void dp_inverse(DpInverseData& d) { - shoc_init(d.nlev, true); - d.transpose(); - dp_inverse_c(d.nlev, d.shcol, d.rho_zt, d.dz_zt, d.rdp_zt); - d.transpose(); -} - -void sfc_fluxes(SfcFluxesData& d) -{ - shoc_init(1, true); // single layer function - d.transpose(); - sfc_fluxes_c(d.shcol, d.num_tracer, d.dtime, d.rho_zi_sfc, d.rdp_zt_sfc, d.wthl_sfc, d.wqw_sfc, d.wtke_sfc, d.wtracer_sfc, d.thetal, d.qw, d.tke, d.wtracer); - d.transpose(); -} - -void impli_srf_stress_term(ImpliSrfStressTermData& d) -{ - shoc_init(1, true); // single layer function - impli_srf_stress_term_c(d.shcol, d.rho_zi_sfc, d.uw_sfc, d.vw_sfc, d.u_wind_sfc, d.v_wind_sfc, d.ksrf); -} - -void tke_srf_flux_term(TkeSrfFluxTermData& d) -{ - shoc_init(1, true); // single layer function - tke_srf_flux_term_c(d.shcol, d.uw_sfc, d.vw_sfc, d.wtke_sfc); + dp_inverse_host(d.nlev, d.shcol, d.rho_zt, d.dz_zt, d.rdp_zt); } void integ_column_stability(IntegColumnStabilityData& d) { - shoc_init(d.nlev, true); - d.transpose(); - integ_column_stability_c(d.nlev, d.shcol, d.dz_zt, d.pres, d.brunt, d.brunt_int); - d.transpose(); + integ_column_stability_host(d.nlev, d.shcol, d.dz_zt, d.pres, d.brunt, d.brunt_int); } void check_tke(CheckTkeData& d) { - shoc_init(d.nlev, true); - d.transpose(); - check_tke_c(d.shcol, d.nlev, d.tke); - d.transpose(); + check_tke_host(d.shcol, d.nlev, d.tke); } void shoc_tke(ShocTkeData& d) { - shoc_init(d.nlev, true); - d.transpose(); - shoc_tke_c(d.shcol, d.nlev, d.nlevi, d.dtime, d.wthv_sec, d.shoc_mix, d.dz_zi, d.dz_zt, d.pres, d.tabs, d.u_wind, d.v_wind, d.brunt, d.zt_grid, d.zi_grid, d.pblh, d.tke, d.tk, d.tkh, d.isotropy); - d.transpose(); + shoc_tke_host(d.shcol, d.nlev, d.nlevi, d.dtime, d.wthv_sec, d.shoc_mix, d.dz_zi, d.dz_zt, d.pres, d.tabs, d.u_wind, d.v_wind, d.brunt, d.zt_grid, d.zi_grid, d.pblh, d.tke, d.tk, d.tkh, d.isotropy); } void compute_shr_prod(ComputeShrProdData& d) { - shoc_init(d.nlev, true); - d.transpose(); - compute_shr_prod_c(d.nlevi, d.nlev, d.shcol, d.dz_zi, d.u_wind, d.v_wind, d.sterm); - d.transpose(); + compute_shr_prod_host(d.nlevi, d.nlev, d.shcol, d.dz_zi, d.u_wind, d.v_wind, d.sterm); } void isotropic_ts(IsotropicTsData& d) { - shoc_init(d.nlev, true); - d.transpose(); - isotropic_ts_c(d.nlev, d.shcol, d.brunt_int, d.tke, d.a_diss, d.brunt, d.isotropy); - d.transpose(); + isotropic_ts_host(d.nlev, d.shcol, d.brunt_int, d.tke, d.a_diss, d.brunt, d.isotropy); } void adv_sgs_tke(AdvSgsTkeData& d) { - shoc_init(d.nlev, true); - d.transpose(); - adv_sgs_tke_c(d.nlev, d.shcol, d.dtime, d.shoc_mix, d.wthv_sec, d.sterm_zt, d.tk, d.tke, d.a_diss); - d.transpose(); + adv_sgs_tke_host(d.nlev, d.shcol, d.dtime, d.shoc_mix, d.wthv_sec, d.sterm_zt, d.tk, d.tke, d.a_diss); } void eddy_diffusivities(EddyDiffusivitiesData& d) { - shoc_init(d.nlev, true); - d.transpose(); - eddy_diffusivities_c(d.nlev, d.shcol, d.pblh, d.zt_grid, d.tabs, d.shoc_mix, d.sterm_zt, d.isotropy, d.tke, d.tkh, d.tk); - d.transpose(); + eddy_diffusivities_host(d.nlev, d.shcol, d.pblh, d.zt_grid, d.tabs, d.shoc_mix, d.sterm_zt, d.isotropy, d.tke, d.tkh, d.tk); } void shoc_length(ShocLengthData& d) { - shoc_init(d.nlev, true); - d.transpose(); - shoc_length_c(d.shcol, d.nlev, d.nlevi, d.host_dx, d.host_dy, d.zt_grid, d.zi_grid, d.dz_zt, d.tke, d.thv, d.brunt, d.shoc_mix); - d.transpose(); + shoc_length_host(d.shcol, d.nlev, d.nlevi, d.host_dx, d.host_dy, d.zt_grid, d.zi_grid, d.dz_zt, d.tke, d.thv, d.brunt, d.shoc_mix); } void compute_brunt_shoc_length(ComputeBruntShocLengthData& d) { - shoc_init(d.nlev, true); - d.transpose(); - compute_brunt_shoc_length_c(d.nlev, d.nlevi, d.shcol, d.dz_zt, d.thv, d.thv_zi, d.brunt); - d.transpose(); + compute_brunt_shoc_length_host(d.nlev, d.nlevi, d.shcol, d.dz_zt, d.thv, d.thv_zi, d.brunt); } void compute_l_inf_shoc_length(ComputeLInfShocLengthData& d) { - shoc_init(d.nlev, true); - d.transpose(); - compute_l_inf_shoc_length_c(d.nlev, d.shcol, d.zt_grid, d.dz_zt, d.tke, d.l_inf); - d.transpose(); + compute_l_inf_shoc_length_host(d.nlev, d.shcol, d.zt_grid, d.dz_zt, d.tke, d.l_inf); } void compute_shoc_mix_shoc_length(ComputeShocMixShocLengthData& d) { - shoc_init(d.nlev, true); - d.transpose(); - compute_shoc_mix_shoc_length_c(d.nlev, d.shcol, d.tke, d.brunt, d.zt_grid, d.l_inf, d.shoc_mix); - d.transpose(); + compute_shoc_mix_shoc_length_host(d.nlev, d.shcol, d.tke, d.brunt, d.zt_grid, d.l_inf, d.shoc_mix); } void check_length_scale_shoc_length(CheckLengthScaleShocLengthData& d) { - shoc_init(d.nlev, true); - d.transpose(); - check_length_scale_shoc_length_c(d.nlev, d.shcol, d.host_dx, d.host_dy, d.shoc_mix); - d.transpose(); -} - -void fterms_input_for_diag_third_shoc_moment(FtermsInputForDiagThirdShocMomentData& d) -{ - shoc_init(1, true); // single level function - fterms_input_for_diag_third_shoc_moment_c(d.dz_zi, d.dz_zt, d.dz_zt_kc, d.isotropy_zi, d.brunt_zi, d.thetal_zi, &d.thedz, &d.thedz2, &d.iso, &d.isosqrd, &d.buoy_sgs2, &d.bet2); -} - -void aa_terms_diag_third_shoc_moment(AaTermsDiagThirdShocMomentData& d) -{ - shoc_init(1, true); // single level function - aa_terms_diag_third_shoc_moment_c(d.omega0, d.omega1, d.omega2, d.x0, d.x1, d.y0, d.y1, &d.aa0, &d.aa1); -} - -void f0_to_f5_diag_third_shoc_moment(F0ToF5DiagThirdShocMomentData& d) -{ - shoc_init(1, true); // single level function - f0_to_f5_diag_third_shoc_moment_c(d.thedz, d.thedz2, d.bet2, d.iso, d.isosqrd, d.wthl_sec, d.wthl_sec_kc, d.wthl_sec_kb, d.thl_sec_kc, d.thl_sec_kb, d.w_sec, d.w_sec_kc, d.w_sec_zi, d.tke, d.tke_kc, &d.f0, &d.f1, &d.f2, &d.f3, &d.f4, &d.f5); -} - -void omega_terms_diag_third_shoc_moment(OmegaTermsDiagThirdShocMomentData& d) -{ - shoc_init(1, true); // single level function - omega_terms_diag_third_shoc_moment_c(d.buoy_sgs2, d.f3, d.f4, &d.omega0, &d.omega1, &d.omega2); -} - -void x_y_terms_diag_third_shoc_moment(XYTermsDiagThirdShocMomentData& d) -{ - shoc_init(1, true); // single level function - x_y_terms_diag_third_shoc_moment_c(d.buoy_sgs2, d.f0, d.f1, d.f2, &d.x0, &d.y0, &d.x1, &d.y1); -} - -void w3_diag_third_shoc_moment(W3DiagThirdShocMomentData& d) -{ - shoc_init(1, true); // single level function - w3_diag_third_shoc_moment_c(d.aa0, d.aa1, d.x0, d.x1, d.f5, &d.w3); + check_length_scale_shoc_length_host(d.nlev, d.shcol, d.host_dx, d.host_dy, d.shoc_mix); } void clipping_diag_third_shoc_moments(ClippingDiagThirdShocMomentsData& d) { - shoc_init(d.nlevi - 1, true); // nlev = nlevi - 1 - d.transpose(); - clipping_diag_third_shoc_moments_c(d.nlevi, d.shcol, d.w_sec_zi, d.w3); - d.transpose(); + clipping_diag_third_shoc_moments_host(d.nlevi, d.shcol, d.w_sec_zi, d.w3); } void diag_second_moments_srf(DiagSecondMomentsSrfData& d) { - shoc_init(1, true); // single level function - shoc_diag_second_moments_srf_c(d.shcol, d.wthl_sfc, d.uw_sfc, d.vw_sfc, d.ustar2, d.wstar); + shoc_diag_second_moments_srf_host(d.shcol, d.wthl_sfc, d.uw_sfc, d.vw_sfc, d.ustar2, d.wstar); } void linear_interp(LinearInterpData& d) { - shoc_init(d.km1, true); - d.transpose(); - linear_interp_c(d.x1, d.x2, d.y1, d.y2, d.km1, d.km2, d.ncol, d.minthresh); - d.transpose(); + linear_interp_host(d.x1, d.x2, d.y1, d.y2, d.km1, d.km2, d.ncol, d.minthresh); } void diag_third_shoc_moments(DiagThirdShocMomentsData& d) { - shoc_init(d.nlev, true); - d.transpose(); - diag_third_shoc_moments_c(d.shcol, d.nlev, d.nlevi, d.w_sec, d.thl_sec, d.wthl_sec, d.isotropy, d.brunt, d.thetal, d.tke, d.dz_zt, d.dz_zi, d.zt_grid, d.zi_grid, d.w3); - d.transpose(); + diag_third_shoc_moments_host(d.shcol, d.nlev, d.nlevi, d.w_sec, d.thl_sec, d.wthl_sec, d.isotropy, d.brunt, d.thetal, d.tke, d.dz_zt, d.dz_zi, d.zt_grid, d.zi_grid, d.w3); } void compute_diag_third_shoc_moment(ComputeDiagThirdShocMomentData& d) { - shoc_init(d.nlev, true); - d.transpose(); - compute_diag_third_shoc_moment_c(d.shcol, d.nlev, d.nlevi, d.w_sec, d.thl_sec, d.wthl_sec, d.tke, d.dz_zt, d.dz_zi, d.isotropy_zi, d.brunt_zi, d.w_sec_zi, d.thetal_zi, d.w3); - d.transpose(); + compute_diag_third_shoc_moment_host(d.shcol, d.nlev, d.nlevi, d.w_sec, d.thl_sec, d.wthl_sec, d.tke, d.dz_zt, d.dz_zi, d.isotropy_zi, d.brunt_zi, d.w_sec_zi, d.thetal_zi, d.w3); } void shoc_assumed_pdf(ShocAssumedPdfData& d) { - shoc_init(d.nlev, true); - d.transpose(); - shoc_assumed_pdf_c(d.shcol, d.nlev, d.nlevi, d.thetal, d.qw, d.w_field, d.thl_sec, d.qw_sec, d.wthl_sec, d.w_sec, d.wqw_sec, d.qwthl_sec, d.w3, d.pres, d.zt_grid, d.zi_grid, d.shoc_cldfrac, d.shoc_ql, d.wqls, d.wthv_sec, d.shoc_ql2); - d.transpose(); + shoc_assumed_pdf_host(d.shcol, d.nlev, d.nlevi, d.thetal, d.qw, d.w_field, d.thl_sec, d.qw_sec, d.wthl_sec, d.w_sec, d.wqw_sec, d.qwthl_sec, d.w3, d.pres, d.zt_grid, d.zi_grid, d.shoc_cldfrac, d.shoc_ql, d.wqls, d.wthv_sec, d.shoc_ql2); } void shoc_assumed_pdf_tilde_to_real(ShocAssumedPdfTildeToRealData& d) { - shoc_init(1, true); // single level function - shoc_assumed_pdf_tilde_to_real_c(d.w_first, d.sqrtw2, &d.w1); + shoc_assumed_pdf_tilde_to_real_host(d.w_first, d.sqrtw2, &d.w1); } void shoc_assumed_pdf_vv_parameters(ShocAssumedPdfVvParametersData& d) { - shoc_init(1, true); // single level function - shoc_assumed_pdf_vv_parameters_c(d.w_first, d.w_sec, d.w3var, &d.skew_w, &d.w1_1, &d.w1_2, &d.w2_1, &d.w2_2, &d.a); + shoc_assumed_pdf_vv_parameters_host(d.w_first, d.w_sec, d.w3var, d.w_tol_sqd, &d.skew_w, &d.w1_1, &d.w1_2, &d.w2_1, &d.w2_2, &d.a); } void shoc_assumed_pdf_thl_parameters(ShocAssumedPdfThlParametersData& d) { - shoc_init(1, true); // single level function - shoc_assumed_pdf_thl_parameters_c(d.wthlsec, d.sqrtw2, d.sqrtthl, d.thlsec, d.thl_first, d.w1_1, d.w1_2, d.skew_w, d.a, d.dothetal_skew, &d.thl1_1, &d.thl1_2, &d.thl2_1, &d.thl2_2, &d.sqrtthl2_1, &d.sqrtthl2_2); + shoc_assumed_pdf_thl_parameters_host(d.wthlsec, d.sqrtw2, d.sqrtthl, d.thlsec, d.thl_first, d.w1_1, d.w1_2, d.skew_w, d.a, d.thl_tol, d.w_thresh, &d.thl1_1, &d.thl1_2, &d.thl2_1, &d.thl2_2, &d.sqrtthl2_1, &d.sqrtthl2_2); } void shoc_assumed_pdf_qw_parameters(ShocAssumedPdfQwParametersData& d) { - shoc_init(1, true); // single level function - shoc_assumed_pdf_qw_parameters_c(d.wqwsec, d.sqrtw2, d.skew_w, d.sqrtqt, d.qwsec, d.w1_2, d.w1_1, d.qw_first, d.a, &d.qw1_1, &d.qw1_2, &d.qw2_1, &d.qw2_2, &d.sqrtqw2_1, &d.sqrtqw2_2); + shoc_assumed_pdf_qw_parameters_host(d.wqwsec, d.sqrtw2, d.skew_w, d.sqrtqt, d.qwsec, d.w1_2, d.w1_1, d.qw_first, d.a, d.rt_tol, d.w_thresh, &d.qw1_1, &d.qw1_2, &d.qw2_1, &d.qw2_2, &d.sqrtqw2_1, &d.sqrtqw2_2); } void shoc_assumed_pdf_inplume_correlations(ShocAssumedPdfInplumeCorrelationsData& d) { - shoc_init(1, true); // single level function - shoc_assumed_pdf_inplume_correlations_c(d.sqrtqw2_1, d.sqrtthl2_1, d.a, d.sqrtqw2_2, d.sqrtthl2_2, d.qwthlsec, d.qw1_1, d.qw_first, d.thl1_1, d.thl_first, d.qw1_2, d.thl1_2, &d.r_qwthl_1); + shoc_assumed_pdf_inplume_correlations_host(d.sqrtqw2_1, d.sqrtthl2_1, d.a, d.sqrtqw2_2, d.sqrtthl2_2, d.qwthlsec, d.qw1_1, d.qw_first, d.thl1_1, d.thl_first, d.qw1_2, d.thl1_2, &d.r_qwthl_1); } void shoc_assumed_pdf_compute_temperature(ShocAssumedPdfComputeTemperatureData& d) { - shoc_init(1, true); // single level function - shoc_assumed_pdf_compute_temperature_c(d.thl1, d.basepres, d.pval, &d.tl1); + shoc_assumed_pdf_compute_temperature_host(d.thl1, d.pval, &d.tl1); } void shoc_assumed_pdf_compute_qs(ShocAssumedPdfComputeQsData& d) { - shoc_init(1, true); // single level function - shoc_assumed_pdf_compute_qs_c(d.tl1_1, d.tl1_2, d.pval, &d.qs1, &d.beta1, &d.qs2, &d.beta2); + shoc_assumed_pdf_compute_qs_host(d.tl1_1, d.tl1_2, d.pval, &d.qs1, &d.beta1, &d.qs2, &d.beta2); } void shoc_assumed_pdf_compute_s(ShocAssumedPdfComputeSData& d) { - shoc_init(1, true); // single level function - shoc_assumed_pdf_compute_s_c(d.qw1, d.qs1, d.beta, d.pval, d.thl2, d.qw2, d.sqrtthl2, d.sqrtqw2, d.r_qwthl, &d.s, &d.std_s, &d.qn, &d.c); + shoc_assumed_pdf_compute_s_host(d.qw1, d.qs1, d.beta, d.pval, d.thl2, d.qw2, d.sqrtthl2, d.sqrtqw2, d.r_qwthl, &d.s, &d.std_s, &d.qn, &d.c); } void shoc_assumed_pdf_compute_sgs_liquid(ShocAssumedPdfComputeSgsLiquidData& d) { - shoc_init(1, true); // single level function - shoc_assumed_pdf_compute_sgs_liquid_c(d.a, d.ql1, d.ql2, &d.shoc_ql); + shoc_assumed_pdf_compute_sgs_liquid_host(d.a, d.ql1, d.ql2, &d.shoc_ql); } void shoc_assumed_pdf_compute_cloud_liquid_variance(ShocAssumedPdfComputeCloudLiquidVarianceData& d) { - shoc_init(1, true); // single level function - shoc_assumed_pdf_compute_cloud_liquid_variance_c(d.a, d.s1, d.ql1, d.c1, d.std_s1, d.s2, d.ql2, d.c2, d.std_s2, d.shoc_ql, &d.shoc_ql2); + shoc_assumed_pdf_compute_cloud_liquid_variance_host(d.a, d.s1, d.ql1, d.c1, d.std_s1, d.s2, d.ql2, d.c2, d.std_s2, d.shoc_ql, &d.shoc_ql2); } void shoc_assumed_pdf_compute_liquid_water_flux(ShocAssumedPdfComputeLiquidWaterFluxData& d) { - shoc_init(1, true); // single level function - shoc_assumed_pdf_compute_liquid_water_flux_c(d.a, d.w1_1, d.w_first, d.ql1, d.w1_2, d.ql2, &d.wqls); + shoc_assumed_pdf_compute_liquid_water_flux_host(d.a, d.w1_1, d.w_first, d.ql1, d.w1_2, d.ql2, &d.wqls); } void shoc_assumed_pdf_compute_buoyancy_flux(ShocAssumedPdfComputeBuoyancyFluxData& d) { - shoc_init(1, true); // single level function - shoc_assumed_pdf_compute_buoyancy_flux_c(d.wthlsec, d.epsterm, d.wqwsec, d.pval, d.wqls, &d.wthv_sec); + shoc_assumed_pdf_compute_buoyancy_flux_host(d.wthlsec, d.wqwsec, d.pval, d.wqls, &d.wthv_sec); } void diag_second_moments_ubycond(DiagSecondMomentsUbycondData& d) { - shoc_init(1, true); // single level function - shoc_diag_second_moments_ubycond_c(d.shcol, d.thl_sec, d.qw_sec, d.wthl_sec, d.wqw_sec, d.qwthl_sec, d.uw_sec, d.vw_sec, d.wtke_sec); + shoc_diag_second_moments_ubycond_host(d.shcol, d.thl_sec, d.qw_sec, d.wthl_sec, d.wqw_sec, d.qwthl_sec, d.uw_sec, d.vw_sec, d.wtke_sec); } void pblintd_init_pot(PblintdInitPotData& d) { - shoc_init(d.nlev, true, true); - d.transpose(); - shoc_pblintd_init_pot_c(d.shcol, d.nlev, d.thl, d.ql, d.q, d.thv); - d.transpose(); + shoc_pblintd_init_pot_host(d.shcol, d.nlev, d.thl, d.ql, d.q, d.thv); } void pblintd_cldcheck(PblintdCldcheckData& d) { - shoc_init(d.nlev, true, true); - d.transpose(); - shoc_pblintd_cldcheck_c(d.shcol, d.nlev, d.nlevi, d.zi, d.cldn, d.pblh); - d.transpose(); + shoc_pblintd_cldcheck_host(d.shcol, d.nlev, d.nlevi, d.zi, d.cldn, d.pblh); } void diag_second_moments_lbycond(DiagSecondMomentsLbycondData& d) { - shoc_init(1, true); // single level function - diag_second_moments_lbycond_c(d.shcol, d.wthl_sfc, d.wqw_sfc, d.uw_sfc, d.vw_sfc, d.ustar2, d.wstar, d.wthl_sec, d.wqw_sec, d.uw_sec, d.vw_sec, d.wtke_sec, d.thl_sec, d.qw_sec, d.qwthl_sec); + diag_second_moments_lbycond_host(d.shcol, d.wthl_sfc, d.wqw_sfc, d.uw_sfc, d.vw_sfc, d.ustar2, d.wstar, d.wthl_sec, d.wqw_sec, d.uw_sec, d.vw_sec, d.wtke_sec, d.thl_sec, d.qw_sec, d.qwthl_sec); } void diag_second_moments(DiagSecondMomentsData& d) { - shoc_init(d.nlev, true); - d.transpose(); - diag_second_moments_c(d.shcol, d.nlev, d.nlevi, d.thetal, d.qw, d.u_wind, d.v_wind, d.tke, d.isotropy, d.tkh, d.tk, - d.dz_zi, d.zt_grid, d.zi_grid, d.shoc_mix, d.thl_sec, d.qw_sec, d.wthl_sec, d.wqw_sec, d.qwthl_sec, d.uw_sec, - d.vw_sec, d.wtke_sec, d.w_sec); - d.transpose(); + diag_second_moments_host(d.shcol, d.nlev, d.nlevi, d.thetal, d.qw, d.u_wind, d.v_wind, d.tke, d.isotropy, d.tkh, d.tk, + d.dz_zi, d.zt_grid, d.zi_grid, d.shoc_mix, d.thl_sec, d.qw_sec, d.wthl_sec, d.wqw_sec, d.qwthl_sec, d.uw_sec, + d.vw_sec, d.wtke_sec, d.w_sec); } void diag_second_shoc_moments(DiagSecondShocMomentsData& d) { - shoc_init(d.nlev, true); - d.transpose(); - diag_second_shoc_moments_c(d.shcol, d.nlev, d.nlevi, d.thetal, d.qw, d.u_wind, d.v_wind, d.tke, d.isotropy, d.tkh, - d.tk, d.dz_zi, d.zt_grid, d.zi_grid, d.shoc_mix, d.wthl_sfc, d.wqw_sfc, d.uw_sfc, d.vw_sfc, d.thl_sec, d.qw_sec, - d.wthl_sec, d.wqw_sec, d.qwthl_sec, d.uw_sec, d.vw_sec, d.wtke_sec, d.w_sec); - d.transpose(); + diag_second_shoc_moments_host(d.shcol, d.nlev, d.nlevi, d.thetal, d.qw, d.u_wind, d.v_wind, d.tke, d.isotropy, d.tkh, + d.tk, d.dz_zi, d.zt_grid, d.zi_grid, d.shoc_mix, d.wthl_sfc, d.wqw_sfc, d.uw_sfc, d.vw_sfc, d.thl_sec, d.qw_sec, + d.wthl_sec, d.wqw_sec, d.qwthl_sec, d.uw_sec, d.vw_sec, d.wtke_sec, d.w_sec); } void compute_shoc_vapor(ComputeShocVaporData& d) { - shoc_init(d.nlev, true); - d.transpose(); - compute_shoc_vapor_c(d.shcol, d.nlev, d.qw, d.ql, d.qv); - d.transpose(); + compute_shoc_vapor_host(d.shcol, d.nlev, d.qw, d.ql, d.qv); } void update_prognostics_implicit(UpdatePrognosticsImplicitData& d) { - shoc_init(d.nlev, true); - d.transpose(); - update_prognostics_implicit_c(d.shcol, d.nlev, d.nlevi, d.num_tracer, d.dtime, - d.dz_zt, d.dz_zi, d.rho_zt, d.zt_grid, d.zi_grid, - d.tk, d.tkh, d.uw_sfc, d.vw_sfc, d.wthl_sfc, d.wqw_sfc, - d.wtracer_sfc, d.thetal, d.qw, d.tracer, d.tke, d.u_wind, d.v_wind); - d.transpose(); + update_prognostics_implicit_host(d.shcol, d.nlev, d.nlevi, d.num_tracer, d.dtime, + d.dz_zt, d.dz_zi, d.rho_zt, d.zt_grid, d.zi_grid, + d.tk, d.tkh, d.uw_sfc, d.vw_sfc, d.wthl_sfc, d.wqw_sfc, + d.wtracer_sfc, d.thetal, d.qw, d.tracer, d.tke, d.u_wind, d.v_wind); } void shoc_main(ShocMainData& d) { - shoc_init(d.nlev, true, true); - d.transpose(); - shoc_main_c(d.shcol, d.nlev, d.nlevi, d.dtime, d.nadv, d.host_dx, d.host_dy, d.thv, d.zt_grid, d.zi_grid, + const int npbl = shoc_init_host(d.nlev, d.pref_mid, d.nbot_shoc, d.ntop_shoc); + d.elapsed_s = shoc_main_host(d.shcol, d.nlev, d.nlevi, d.dtime, d.nadv, npbl, d.host_dx, d.host_dy, d.thv, d.zt_grid, d.zi_grid, d.pres, d.presi, d.pdel, d.wthl_sfc, d.wqw_sfc, d.uw_sfc, d.vw_sfc, d.wtracer_sfc, d.num_qtracers, d.w_field, d.inv_exner, d.phis, d.host_dse, d.tke, d.thetal, d.qw, d.u_wind, d.v_wind, d.qtracers, d.wthv_sec, d.tkh, d.tk, d.shoc_ql, d.shoc_cldfrac, d.pblh, d.shoc_mix, d.isotropy, d.w_sec, d.thl_sec, d.qw_sec, d.qwthl_sec, d.wthl_sec, d.wqw_sec, - d.wtke_sec, d.uw_sec, d.vw_sec, d.w3, d.wqls_sec, d.brunt, d.shoc_ql2, &d.elapsed_s); - d.transpose(); -} - -void shoc_main_with_init(ShocMainData& d) -{ - using C = scream::physics::Constants; - - d.transpose(); - shoc_init_for_main_bfb_c(d.nlev, C::gravit, C::Rair, C::RH2O, C::Cpair, C::ZVIR, C::LatVap, C::LatIce, C::Karman, C::P0, - d.pref_mid, d.nbot_shoc, d.ntop_shoc+1); - shoc_use_cxx_c(false); - - - shoc_main_c(d.shcol, d.nlev, d.nlevi, d.dtime, d.nadv, d.host_dx, d.host_dy, d.thv, d.zt_grid, d.zi_grid, - d.pres, d.presi, d.pdel, d.wthl_sfc, d.wqw_sfc, d.uw_sfc, d.vw_sfc, d.wtracer_sfc, d.num_qtracers, - d.w_field, d.inv_exner, d.phis, d.host_dse, d.tke, d.thetal, d.qw, d.u_wind, d.v_wind, d.qtracers, - d.wthv_sec, d.tkh, d.tk, d.shoc_ql, d.shoc_cldfrac, d.pblh, d.shoc_mix, d.isotropy, d.w_sec, - d.thl_sec, d.qw_sec, d.qwthl_sec, d.wthl_sec, d.wqw_sec, d.wtke_sec, d.uw_sec, d.vw_sec, d.w3, - d.wqls_sec, d.brunt, d.shoc_ql2, &d.elapsed_s); - d.transpose(); + d.wtke_sec, d.uw_sec, d.vw_sec, d.w3, d.wqls_sec, d.brunt, d.shoc_ql2); } void pblintd_height(PblintdHeightData& d) { - shoc_init(d.nlev, true, true); - d.transpose(); - pblintd_height_c(d.shcol, d.nlev, d.npbl, d.z, d.u, d.v, d.ustar, d.thv, d.thv_ref, d.pblh, d.rino, d.check); - d.transpose(); + pblintd_height_host(d.shcol, d.nlev, d.npbl, d.z, d.u, d.v, d.ustar, d.thv, d.thv_ref, d.pblh, d.rino, d.check); } void vd_shoc_decomp_and_solve(VdShocDecompandSolveData& d) { - shoc_init(d.nlev, true); - d.transpose(); - // Call decomp subroutine - vd_shoc_decomp_c(d.shcol, d.nlev, d.nlevi, d.kv_term, d.tmpi, d.rdp_zt, d.dtime, d.flux, d.du, d.dl, d.d); - // Call solver for each problem. The `var` array represents 3d - // data with an entry per (shcol, nlev, n_rhs). Fortran requires - // 2d data (shcol, nlev) for each rhs. - const Int size = d.shcol*d.nlev; - for (Int n=0; n(); + vd_shoc_decomp_and_solve_host(d.shcol, d.nlev, d.nlevi, d.n_rhs, d.dtime, d.kv_term, d.tmpi, d.rdp_zt, d.flux, d.var); } void pblintd_surf_temp(PblintdSurfTempData& d) { - shoc_init(d.nlev, true, true); - d.transpose(); - pblintd_surf_temp_c(d.shcol, d.nlev, d.nlevi, d.z, d.ustar, d.obklen, d.kbfs, d.thv, d.tlv, d.pblh, d.check, d.rino); - d.transpose(); + pblintd_surf_temp_host(d.shcol, d.nlev, d.nlevi, d.z, d.ustar, d.obklen, d.kbfs, d.thv, d.tlv, d.pblh, d.check, d.rino); } void pblintd_check_pblh(PblintdCheckPblhData& d) { - shoc_init(d.nlev, true, true); - d.transpose(); - pblintd_check_pblh_c(d.shcol, d.nlev, d.nlevi, d.z, d.ustar, d.check, d.pblh); - d.transpose(); + pblintd_check_pblh_host(d.shcol, d.nlev, d.nlevi, d.nlev/*npbl*/, d.z, d.ustar, d.check, d.pblh); } void pblintd(PblintdData& d) { - shoc_init(d.nlev, true, true); - d.transpose(); - pblintd_c(d.shcol, d.nlev, d.nlevi, d.npbl, d.z, d.zi, d.thl, d.ql, d.q, d.u, d.v, d.ustar, d.obklen, d.kbfs, d.cldn, d.pblh); - d.transpose(); + pblintd_host(d.shcol, d.nlev, d.nlevi, d.npbl, d.z, d.zi, d.thl, d.ql, d.q, d.u, d.v, d.ustar, d.obklen, d.kbfs, d.cldn, d.pblh); } void compute_shoc_temperature(ComputeShocTempData& d) { - shoc_init(d.nlev, true, true); - d.transpose(); - compute_shoc_temperature_c(d.shcol, d.nlev, d.thetal, d.ql, d.inv_exner, d.tabs); - d.transpose(); + compute_shoc_temperature_host(d.shcol, d.nlev, d.thetal, d.ql, d.inv_exner, d.tabs); } // end _c impls // -// _f function definitions. These expect data in C layout +// _host function definitions. These expect data in C layout // -void calc_shoc_varorcovar_f(Int shcol, Int nlev, Int nlevi, Real tunefac, +void calc_shoc_varorcovar_host(Int shcol, Int nlev, Int nlevi, Real tunefac, Real *isotropy_zi, Real *tkh_zi, Real *dz_zi, Real *invar1, Real *invar2, Real *varorcovar) { @@ -861,7 +332,7 @@ void calc_shoc_varorcovar_f(Int shcol, Int nlev, Int nlevi, Real tunefac, std::vector ptr_array = {isotropy_zi, tkh_zi, dz_zi, invar1, invar2, varorcovar}; // Sync to device - ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d, true); + ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d); view_2d isotropy_zi_d (temp_d[0]), @@ -889,10 +360,10 @@ void calc_shoc_varorcovar_f(Int shcol, Int nlev, Int nlevi, Real tunefac, // Sync back to host std::vector inout_views = {varorcovar_d}; - ekat::device_to_host({varorcovar}, shcol, nlevi, inout_views, true); + ekat::device_to_host({varorcovar}, shcol, nlevi, inout_views); } -void calc_shoc_vertflux_f(Int shcol, Int nlev, Int nlevi, Real *tkh_zi, +void calc_shoc_vertflux_host(Int shcol, Int nlev, Int nlevi, Real *tkh_zi, Real *dz_zi, Real *invar, Real *vertflux) { using SHF = Functions; @@ -911,7 +382,7 @@ void calc_shoc_vertflux_f(Int shcol, Int nlev, Int nlevi, Real *tkh_zi, std::vector ptr_array = {tkh_zi, dz_zi, invar, vertflux}; // Sync to device - ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d, true); + ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d); view_2d tkh_zi_d (temp_d[0]), @@ -934,10 +405,10 @@ void calc_shoc_vertflux_f(Int shcol, Int nlev, Int nlevi, Real *tkh_zi, // Sync back to host std::vector inout_views = {vertflux_d}; - ekat::device_to_host({vertflux}, shcol, nlevi, inout_views, true); + ekat::device_to_host({vertflux}, shcol, nlevi, inout_views); } -void shoc_diag_second_moments_srf_f(Int shcol, Real* wthl_sfc, Real* uw_sfc, Real* vw_sfc, Real* ustar2, Real* wstar) +void shoc_diag_second_moments_srf_host(Int shcol, Real* wthl_sfc, Real* uw_sfc, Real* vw_sfc, Real* ustar2, Real* wstar) { using SHOC = Functions; using Scalar = typename SHOC::Scalar; @@ -975,7 +446,7 @@ void shoc_diag_second_moments_srf_f(Int shcol, Real* wthl_sfc, Real* uw_sfc, Rea ScreamDeepCopy::copy_to_host({ustar2, wstar}, shcol, inout_views); } -void shoc_diag_second_moments_ubycond_f(Int shcol, Real* thl_sec, Real* qw_sec, Real* wthl_sec, Real* wqw_sec, Real* qwthl_sec, Real* uw_sec, Real* vw_sec, +void shoc_diag_second_moments_ubycond_host(Int shcol, Real* thl_sec, Real* qw_sec, Real* wthl_sec, Real* wqw_sec, Real* qwthl_sec, Real* uw_sec, Real* vw_sec, Real* wtke_sec) { using SHOC = Functions; @@ -1020,7 +491,7 @@ void shoc_diag_second_moments_ubycond_f(Int shcol, Real* thl_sec, Real* qw_sec, ScreamDeepCopy::copy_to_host({thl_sec, qw_sec, qwthl_sec, wthl_sec, wqw_sec, uw_sec, vw_sec, wtke_sec}, shcol, host_views); } -void update_host_dse_f(Int shcol, Int nlev, Real* thlm, Real* shoc_ql, Real* inv_exner, Real* zt_grid, +void update_host_dse_host(Int shcol, Int nlev, Real* thlm, Real* shoc_ql, Real* inv_exner, Real* zt_grid, Real* phis, Real* host_dse) { using SHF = Functions; @@ -1039,7 +510,7 @@ void update_host_dse_f(Int shcol, Int nlev, Real* thlm, Real* shoc_ql, Real* inv // Sync to device ScreamDeepCopy::copy_to_device({phis}, shcol, temp_1d_d); - ekat::host_to_device(ptr_array, shcol, nlev, temp_2d_d, true); + ekat::host_to_device(ptr_array, shcol, nlev, temp_2d_d); view_1d phis_d(temp_1d_d[0]); @@ -1067,10 +538,10 @@ void update_host_dse_f(Int shcol, Int nlev, Real* thlm, Real* shoc_ql, Real* inv // Sync back to host std::vector inout_views = {host_dse_d}; - ekat::device_to_host({host_dse}, shcol, nlev, inout_views, true); + ekat::device_to_host({host_dse}, shcol, nlev, inout_views); } -void compute_diag_third_shoc_moment_f(Int shcol, Int nlev, Int nlevi, Real* w_sec, +void compute_diag_third_shoc_moment_host(Int shcol, Int nlev, Int nlevi, Real* w_sec, Real* thl_sec, Real* wthl_sec, Real* tke, Real* dz_zt, Real* dz_zi, Real* isotropy_zi, Real* brunt_zi, Real* w_sec_zi, Real* thetal_zi, @@ -1100,7 +571,7 @@ void compute_diag_third_shoc_moment_f(Int shcol, Int nlev, Int nlevi, Real* w_se w3}; // Sync to device - ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d, true); + ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d); view_2d w_sec_d (temp_d[0]), @@ -1141,10 +612,10 @@ void compute_diag_third_shoc_moment_f(Int shcol, Int nlev, Int nlevi, Real* w_se // Sync back to host std::vector inout_views = {w3_d}; - ekat::device_to_host({w3}, shcol, nlevi, inout_views, true); + ekat::device_to_host({w3}, shcol, nlevi, inout_views); } -void shoc_pblintd_init_pot_f(Int shcol, Int nlev, Real *thl, Real* ql, Real* q, +void shoc_pblintd_init_pot_host(Int shcol, Int nlev, Real *thl, Real* ql, Real* q, Real *thv) { using SHOC = Functions; @@ -1157,7 +628,7 @@ void shoc_pblintd_init_pot_f(Int shcol, Int nlev, Real *thl, Real* ql, Real* q, static constexpr Int num_arrays = 3; std::vector temp_d(num_arrays); - ekat::host_to_device({thl, ql, q}, shcol, nlev, temp_d, true); + ekat::host_to_device({thl, ql, q}, shcol, nlev, temp_d); view_2d thl_d(temp_d[0]), ql_d (temp_d[1]), @@ -1179,10 +650,10 @@ void shoc_pblintd_init_pot_f(Int shcol, Int nlev, Real *thl, Real* ql, Real* q, }); std::vector inout_views = {thv_d}; - ekat::device_to_host({thv}, shcol, nlev, inout_views, true); + ekat::device_to_host({thv}, shcol, nlev, inout_views); } -void compute_shoc_mix_shoc_length_f(Int nlev, Int shcol, Real* tke, Real* brunt, +void compute_shoc_mix_shoc_length_host(Int nlev, Int shcol, Real* tke, Real* brunt, Real* zt_grid, Real* l_inf, Real* shoc_mix) { using SHF = Functions; @@ -1201,7 +672,7 @@ void compute_shoc_mix_shoc_length_f(Int nlev, Int shcol, Real* tke, Real* brunt, // Sync to device ScreamDeepCopy::copy_to_device({l_inf}, shcol, temp_1d_d); - ekat::host_to_device(ptr_array, shcol, nlev, temp_2d_d, true); + ekat::host_to_device(ptr_array, shcol, nlev, temp_2d_d); view_1d l_inf_d (temp_1d_d[0]); @@ -1231,10 +702,10 @@ void compute_shoc_mix_shoc_length_f(Int nlev, Int shcol, Real* tke, Real* brunt, // Sync back to host std::vector inout_views = {shoc_mix_d}; - ekat::device_to_host({shoc_mix}, shcol, nlev, inout_views, true); + ekat::device_to_host({shoc_mix}, shcol, nlev, inout_views); } -void check_tke_f(Int shcol, Int nlev, Real* tke) +void check_tke_host(Int shcol, Int nlev, Real* tke) { using SHOC = Functions; using Spack = typename SHOC::Spack; @@ -1246,7 +717,7 @@ void check_tke_f(Int shcol, Int nlev, Real* tke) std::vector temp_2d_d(1); // Sync to device - ekat::host_to_device({tke}, shcol, nlev, temp_2d_d, true); + ekat::host_to_device({tke}, shcol, nlev, temp_2d_d); view_2d tke_d(temp_2d_d[0]); @@ -1263,10 +734,10 @@ void check_tke_f(Int shcol, Int nlev, Real* tke) // Sync back to host std::vector inout_views = {tke_d}; - ekat::device_to_host({tke}, shcol, nlev, inout_views, true); + ekat::device_to_host({tke}, shcol, nlev, inout_views); } -void linear_interp_f(Real* x1, Real* x2, Real* y1, Real* y2, Int km1, Int km2, Int ncol, Real minthresh) +void linear_interp_host(Real* x1, Real* x2, Real* y1, Real* y2, Int km1, Int km2, Int ncol, Real minthresh) { using SHF = Functions; @@ -1282,7 +753,7 @@ void linear_interp_f(Real* x1, Real* x2, Real* y1, Real* y2, Int km1, Int km2, I std::vector ptr_array = {x1, x2, y1}; // Sync to device - ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_2d_d, true); + ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_2d_d); view_2d x1_d(temp_2d_d[0]), @@ -1305,10 +776,10 @@ void linear_interp_f(Real* x1, Real* x2, Real* y1, Real* y2, Int km1, Int km2, I // Sync back to host std::vector inout_views = {y2_d}; - ekat::device_to_host({y2}, ncol, km2, inout_views, true); + ekat::device_to_host({y2}, ncol, km2, inout_views); } -void clipping_diag_third_shoc_moments_f(Int nlevi, Int shcol, Real *w_sec_zi, +void clipping_diag_third_shoc_moments_host(Int nlevi, Int shcol, Real *w_sec_zi, Real *w3) { using SHF = Functions; @@ -1321,7 +792,7 @@ void clipping_diag_third_shoc_moments_f(Int nlevi, Int shcol, Real *w_sec_zi, // Sync to device std::vector temp_d(2); - ekat::host_to_device({w_sec_zi, w3}, shcol, nlevi, temp_d, true); + ekat::host_to_device({w_sec_zi, w3}, shcol, nlevi, temp_d); view_2d w_sec_zi_d(temp_d[0]), @@ -1340,10 +811,10 @@ void clipping_diag_third_shoc_moments_f(Int nlevi, Int shcol, Real *w_sec_zi, // Sync back to host std::vector inout_views = {w3_d}; - ekat::device_to_host({w3}, shcol, nlevi, inout_views, true); + ekat::device_to_host({w3}, shcol, nlevi, inout_views); } -void shoc_energy_integrals_f(Int shcol, Int nlev, Real *host_dse, Real *pdel, +void shoc_energy_integrals_host(Int shcol, Int nlev, Real *host_dse, Real *pdel, Real *rtm, Real *rcm, Real *u_wind, Real *v_wind, Real *se_int, Real *ke_int, Real *wv_int, Real *wl_int) { @@ -1361,7 +832,7 @@ void shoc_energy_integrals_f(Int shcol, Int nlev, Real *host_dse, Real *pdel, std::vector ptr_array = {host_dse, pdel, rtm, rcm, u_wind, v_wind}; // Sync to device - ekat::host_to_device(ptr_array, shcol, nlev, temp_d, true); + ekat::host_to_device(ptr_array, shcol, nlev, temp_d); // inputs view_2d @@ -1410,7 +881,7 @@ void shoc_energy_integrals_f(Int shcol, Int nlev, Real *host_dse, Real *pdel, ScreamDeepCopy::copy_to_host({se_int,ke_int,wv_int,wl_int}, shcol, inout_views); } -void diag_second_moments_lbycond_f(Int shcol, Real* wthl_sfc, Real* wqw_sfc, Real* uw_sfc, Real* vw_sfc, Real* ustar2, Real* wstar, +void diag_second_moments_lbycond_host(Int shcol, Real* wthl_sfc, Real* wqw_sfc, Real* uw_sfc, Real* vw_sfc, Real* ustar2, Real* wstar, Real* wthl_sec, Real* wqw_sec, Real* uw_sec, Real* vw_sec, Real* wtke_sec, Real* thl_sec, Real* qw_sec, Real* qwthl_sec) { using SHOC = Functions; @@ -1473,7 +944,7 @@ void diag_second_moments_lbycond_f(Int shcol, Real* wthl_sfc, Real* wqw_sfc, Rea ScreamDeepCopy::copy_to_host({wthl_sec, wqw_sec, uw_sec, vw_sec, wtke_sec, thl_sec, qw_sec, qwthl_sec}, shcol, host_views); } -void diag_second_moments_f(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* qw, Real* u_wind, Real* v_wind, +void diag_second_moments_host(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* qw, Real* u_wind, Real* v_wind, Real* tke, Real* isotropy, Real* tkh, Real* tk, Real* dz_zi, Real* zt_grid, Real* zi_grid, Real* shoc_mix, Real* thl_sec, Real* qw_sec, Real* wthl_sec, Real* wqw_sec, Real* qwthl_sec, Real* uw_sec, Real* vw_sec, Real* wtke_sec, Real* w_sec) @@ -1493,7 +964,7 @@ void diag_second_moments_f(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* q std::vector ptr_array = {thetal, qw, u_wind, v_wind, tke, isotropy, tkh, tk, zt_grid, shoc_mix, thl_sec, qw_sec, wthl_sec, wqw_sec, qwthl_sec, uw_sec, vw_sec, wtke_sec, dz_zi, zi_grid}; - ekat::host_to_device(ptr_array, dim1_array, dim2_array, temp_2d, true); + ekat::host_to_device(ptr_array, dim1_array, dim2_array, temp_2d); view_2d thetal_2d (temp_2d[0]), @@ -1571,10 +1042,10 @@ void diag_second_moments_f(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* q std::vector dim1(9, shcol); std::vector dim2 = {nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlev }; std::vector host_views = {thl_sec_2d, qw_sec_2d, wthl_sec_2d, wqw_sec_2d, qwthl_sec_2d, uw_sec_2d, vw_sec_2d, wtke_sec_2d, w_sec_2d}; - ekat::device_to_host({thl_sec, qw_sec, wthl_sec, wqw_sec, qwthl_sec, uw_sec, vw_sec, wtke_sec, w_sec}, dim1, dim2, host_views, true); + ekat::device_to_host({thl_sec, qw_sec, wthl_sec, wqw_sec, qwthl_sec, uw_sec, vw_sec, wtke_sec, w_sec}, dim1, dim2, host_views); } -void diag_second_shoc_moments_f(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* qw, Real* u_wind, Real* v_wind, Real* tke, +void diag_second_shoc_moments_host(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* qw, Real* u_wind, Real* v_wind, Real* tke, Real* isotropy, Real* tkh, Real* tk, Real* dz_zi, Real* zt_grid, Real* zi_grid, Real* shoc_mix, Real* wthl_sfc, Real* wqw_sfc, Real* uw_sfc, Real* vw_sfc, Real* thl_sec, Real* qw_sec, Real* wthl_sec, Real* wqw_sec, Real* qwthl_sec, Real* uw_sec, Real* vw_sec, Real* wtke_sec, Real* w_sec) @@ -1603,7 +1074,7 @@ void diag_second_shoc_moments_f(Int shcol, Int nlev, Int nlevi, Real* thetal, Re std::vector ptr_array = {thetal, qw, u_wind, v_wind, tke, isotropy, tkh, tk, zt_grid, shoc_mix, thl_sec, qw_sec, wthl_sec, wqw_sec, qwthl_sec, uw_sec, vw_sec, wtke_sec, dz_zi, zi_grid}; - ekat::host_to_device(ptr_array, dim1_array, dim2_array, temp_2d, true); + ekat::host_to_device(ptr_array, dim1_array, dim2_array, temp_2d); view_2d thetal_2d (temp_2d[0]), @@ -1689,10 +1160,10 @@ void diag_second_shoc_moments_f(Int shcol, Int nlev, Int nlevi, Real* thetal, Re std::vector dim1(9, shcol); std::vector dim2 = {nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlev }; std::vector host_2d_views = {thl_sec_2d, qw_sec_2d, wthl_sec_2d, wqw_sec_2d, qwthl_sec_2d, uw_sec_2d, vw_sec_2d, wtke_sec_2d, w_sec_2d}; - ekat::device_to_host({thl_sec, qw_sec, wthl_sec, wqw_sec, qwthl_sec, uw_sec, vw_sec, wtke_sec, w_sec}, dim1, dim2, host_2d_views, true); + ekat::device_to_host({thl_sec, qw_sec, wthl_sec, wqw_sec, qwthl_sec, uw_sec, vw_sec, wtke_sec, w_sec}, dim1, dim2, host_2d_views); } -void compute_brunt_shoc_length_f(Int nlev, Int nlevi, Int shcol, Real* dz_zt, Real* thv, Real* thv_zi, Real* brunt) +void compute_brunt_shoc_length_host(Int nlev, Int nlevi, Int shcol, Real* dz_zt, Real* thv, Real* thv_zi, Real* brunt) { using SHF = Functions; @@ -1708,7 +1179,7 @@ void compute_brunt_shoc_length_f(Int nlev, Int nlevi, Int shcol, Real* dz_zt, Re std::vector ptr_array = {dz_zt, thv, thv_zi, brunt}; // Sync to device - ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d, true); + ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d); view_2d dz_zt_d (temp_d[0]), @@ -1731,10 +1202,10 @@ void compute_brunt_shoc_length_f(Int nlev, Int nlevi, Int shcol, Real* dz_zt, Re // Sync back to host std::vector inout_views = {brunt_d}; - ekat::device_to_host({brunt}, shcol, nlev, inout_views, true); + ekat::device_to_host({brunt}, shcol, nlev, inout_views); } -void compute_l_inf_shoc_length_f(Int nlev, Int shcol, Real *zt_grid, Real *dz_zt, +void compute_l_inf_shoc_length_host(Int nlev, Int shcol, Real *zt_grid, Real *dz_zt, Real *tke, Real *l_inf) { using SHF = Functions; @@ -1749,7 +1220,7 @@ void compute_l_inf_shoc_length_f(Int nlev, Int shcol, Real *zt_grid, Real *dz_zt // Sync to device std::vector temp_d(3); - ekat::host_to_device({zt_grid, dz_zt, tke}, shcol, nlev, temp_d, true); + ekat::host_to_device({zt_grid, dz_zt, tke}, shcol, nlev, temp_d); // inputs view_2d @@ -1782,7 +1253,7 @@ void compute_l_inf_shoc_length_f(Int nlev, Int shcol, Real *zt_grid, Real *dz_zt ScreamDeepCopy::copy_to_host({l_inf}, shcol, inout_views); } -void check_length_scale_shoc_length_f(Int nlev, Int shcol, Real* host_dx, Real* host_dy, Real* shoc_mix) +void check_length_scale_shoc_length_host(Int nlev, Int shcol, Real* host_dx, Real* host_dy, Real* shoc_mix) { using SHF = Functions; @@ -1798,7 +1269,7 @@ void check_length_scale_shoc_length_f(Int nlev, Int shcol, Real* host_dx, Real* std::vector temp_1d_d(2); std::vector temp_2d_d(1); ScreamDeepCopy::copy_to_device({host_dx,host_dy}, shcol, temp_1d_d); - ekat::host_to_device({shoc_mix}, shcol, nlev, temp_2d_d, true); + ekat::host_to_device({shoc_mix}, shcol, nlev, temp_2d_d); view_1d host_dx_d(temp_1d_d[0]), @@ -1821,10 +1292,10 @@ void check_length_scale_shoc_length_f(Int nlev, Int shcol, Real* host_dx, Real* // Sync back to host std::vector inout_views = {shoc_mix_d}; - ekat::device_to_host({shoc_mix}, shcol, nlev, inout_views, true); + ekat::device_to_host({shoc_mix}, shcol, nlev, inout_views); } -void shoc_diag_obklen_f(Int shcol, Real* uw_sfc, Real* vw_sfc, Real* wthl_sfc, Real* wqw_sfc, Real* thl_sfc, +void shoc_diag_obklen_host(Int shcol, Real* uw_sfc, Real* vw_sfc, Real* wthl_sfc, Real* wqw_sfc, Real* thl_sfc, Real* cldliq_sfc, Real* qv_sfc, Real* ustar, Real* kbfs, Real* obklen) { using SHF = Functions; @@ -1880,7 +1351,7 @@ void shoc_diag_obklen_f(Int shcol, Real* uw_sfc, Real* vw_sfc, Real* wthl_sfc, R ScreamDeepCopy::copy_to_host({ustar, kbfs, obklen}, shcol, inout_views); } -void shoc_pblintd_cldcheck_f(Int shcol, Int nlev, Int nlevi, Real* zi, Real* cldn, Real* pblh) { +void shoc_pblintd_cldcheck_host(Int shcol, Int nlev, Int nlevi, Real* zi, Real* cldn, Real* pblh) { using SHOC = Functions; using Spack = typename SHOC::Spack; using Scalar = typename SHOC::Scalar; @@ -1891,7 +1362,7 @@ void shoc_pblintd_cldcheck_f(Int shcol, Int nlev, Int nlevi, Real* zi, Real* cld std::vector dim2 = {nlevi, nlev}; std::vector cldcheck_2d(2); - ekat::host_to_device({zi, cldn}, dim1, dim2, cldcheck_2d, true); + ekat::host_to_device({zi, cldn}, dim1, dim2, cldcheck_2d); view_2d zi_2d (cldcheck_2d[0]), @@ -1919,7 +1390,7 @@ void shoc_pblintd_cldcheck_f(Int shcol, Int nlev, Int nlevi, Real* zi, Real* cld ScreamDeepCopy::copy_to_host({pblh}, shcol, inout_views); } -void shoc_length_f(Int shcol, Int nlev, Int nlevi, Real* host_dx, Real* host_dy, +void shoc_length_host(Int shcol, Int nlev, Int nlevi, Real* host_dx, Real* host_dy, Real* zt_grid, Real* zi_grid, Real*dz_zt, Real* tke, Real* thv, Real*brunt, Real* shoc_mix) { @@ -1942,7 +1413,7 @@ void shoc_length_f(Int shcol, Int nlev, Int nlevi, Real* host_dx, Real* host_dy, thv, brunt, shoc_mix}; // Sync to device ScreamDeepCopy::copy_to_device({host_dx, host_dy}, shcol, temp_1d_d); - ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_2d_d, true); + ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_2d_d); // inputs view_1d @@ -1991,10 +1462,10 @@ void shoc_length_f(Int shcol, Int nlev, Int nlevi, Real* host_dx, Real* host_dy, // Sync back to host std::vector inout_views = {brunt_d,shoc_mix_d}; - ekat::device_to_host({brunt,shoc_mix}, shcol, nlev, inout_views, true); + ekat::device_to_host({brunt,shoc_mix}, shcol, nlev, inout_views); } -void shoc_energy_fixer_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Real* zt_grid, +void shoc_energy_fixer_host(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Real* zt_grid, Real* zi_grid, Real* se_b, Real* ke_b, Real* wv_b, Real* wl_b, Real* se_a, Real* ke_a, Real* wv_a, Real* wl_a, Real* wthl_sfc, Real* wqw_sfc, Real* rho_zt, Real* tke, Real* pint, @@ -2022,7 +1493,7 @@ void shoc_energy_fixer_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, R // Sync to device ScreamDeepCopy::copy_to_device(ptr_array_1d, shcol, temp_1d_d); - ekat::host_to_device(ptr_array_2d, dim1_sizes, dim2_sizes, temp_2d_d, true); + ekat::host_to_device(ptr_array_2d, dim1_sizes, dim2_sizes, temp_2d_d); view_1d se_b_d(temp_1d_d[0]), @@ -2082,10 +1553,10 @@ void shoc_energy_fixer_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, R // Sync back to host std::vector inout_views = {host_dse_d}; - ekat::device_to_host({host_dse}, shcol, nlev, inout_views, true); + ekat::device_to_host({host_dse}, shcol, nlev, inout_views); } -void compute_shoc_vapor_f(Int shcol, Int nlev, Real* qw, Real* ql, Real* qv) +void compute_shoc_vapor_host(Int shcol, Int nlev, Real* qw, Real* ql, Real* qv) { using SHF = Functions; @@ -2099,7 +1570,7 @@ void compute_shoc_vapor_f(Int shcol, Int nlev, Real* qw, Real* ql, Real* qv) // Sync to device std::vector temp_d(num_arrays); - ekat::host_to_device( {qw, ql, qv}, shcol, nlev, temp_d, true); + ekat::host_to_device( {qw, ql, qv}, shcol, nlev, temp_d); // Inputs/Outputs view_2d @@ -2121,10 +1592,10 @@ void compute_shoc_vapor_f(Int shcol, Int nlev, Real* qw, Real* ql, Real* qv) // Sync back to host std::vector inout_views = {qv_d}; - ekat::device_to_host({qv}, shcol, nlev, inout_views, true); + ekat::device_to_host({qv}, shcol, nlev, inout_views); } -void update_prognostics_implicit_f(Int shcol, Int nlev, Int nlevi, Int num_tracer, Real dtime, +void update_prognostics_implicit_host(Int shcol, Int nlev, Int nlevi, Int num_tracer, Real dtime, Real* dz_zt, Real* dz_zi, Real* rho_zt, Real* zt_grid, Real* zi_grid, Real* tk, Real* tkh, Real* uw_sfc, Real* vw_sfc, Real* wthl_sfc, Real* wqw_sfc, Real* wtracer_sfc, Real* thetal, Real* qw, Real* tracer, @@ -2157,8 +1628,8 @@ void update_prognostics_implicit_f(Int shcol, Int nlev, Int nlevi, Int num_trace // Sync to device ScreamDeepCopy::copy_to_device({uw_sfc, vw_sfc, wthl_sfc, wqw_sfc}, shcol, temp_1d_d); - ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_2d_d, true); - ekat::host_to_device({tracer}, shcol, nlev, num_tracer, temp_3d_d, true); + ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_2d_d); + ekat::host_to_device({tracer}, shcol, nlev, num_tracer, temp_3d_d); view_1d uw_sfc_d(temp_1d_d[0]), @@ -2256,13 +1727,13 @@ void update_prognostics_implicit_f(Int shcol, Int nlev, Int nlevi, Int num_trace // Sync back to host std::vector inout_views_2d = {thetal_d, qw_d, u_wind_d, v_wind_d, tke_d}; - ekat::device_to_host({thetal, qw, u_wind, v_wind, tke}, shcol, nlev, inout_views_2d, true); + ekat::device_to_host({thetal, qw, u_wind, v_wind, tke}, shcol, nlev, inout_views_2d); std::vector inout_views = {qtracers_f90_d}; - ekat::device_to_host({tracer}, shcol, nlev, num_tracer, inout_views, true); + ekat::device_to_host({tracer}, shcol, nlev, num_tracer, inout_views); } -void diag_third_shoc_moments_f(Int shcol, Int nlev, Int nlevi, Real* w_sec, Real* thl_sec, +void diag_third_shoc_moments_host(Int shcol, Int nlev, Int nlevi, Real* w_sec, Real* thl_sec, Real* wthl_sec, Real* isotropy, Real* brunt, Real* thetal, Real* tke, Real* dz_zt, Real* dz_zi, Real* zt_grid, Real* zi_grid, Real* w3) @@ -2285,7 +1756,7 @@ void diag_third_shoc_moments_f(Int shcol, Int nlev, Int nlevi, Real* w_sec, Real dz_zi, zt_grid, zi_grid, w3}; // Sync to device - ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d, true); + ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d); view_2d wsec_d(temp_d[0]), @@ -2338,10 +1809,10 @@ void diag_third_shoc_moments_f(Int shcol, Int nlev, Int nlevi, Real* w_sec, Real // Sync back to host std::vector inout_views = {w3_d}; - ekat::device_to_host({w3}, shcol, nlevi, inout_views, true); + ekat::device_to_host({w3}, shcol, nlevi, inout_views); } -void adv_sgs_tke_f(Int nlev, Int shcol, Real dtime, Real* shoc_mix, Real* wthv_sec, +void adv_sgs_tke_host(Int nlev, Int shcol, Real dtime, Real* shoc_mix, Real* wthv_sec, Real* sterm_zt, Real* tk, Real* tke, Real* a_diss) { using SHF = Functions; @@ -2358,7 +1829,7 @@ void adv_sgs_tke_f(Int nlev, Int shcol, Real dtime, Real* shoc_mix, Real* wthv_s std::vector ptr_array = {shoc_mix, wthv_sec, sterm_zt, tk, tke, a_diss}; // Sync to device - ekat::host_to_device(ptr_array, shcol, nlev, temp_d, true); + ekat::host_to_device(ptr_array, shcol, nlev, temp_d); view_2d //input @@ -2389,10 +1860,10 @@ void adv_sgs_tke_f(Int nlev, Int shcol, Real dtime, Real* shoc_mix, Real* wthv_s // Sync back to host std::vector inout_views = {tke_d, a_diss_d}; - ekat::device_to_host({tke, a_diss}, shcol, nlev, inout_views, true); + ekat::device_to_host({tke, a_diss}, shcol, nlev, inout_views); } -void shoc_assumed_pdf_f(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* qw, Real* w_field, +void shoc_assumed_pdf_host(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* qw, Real* w_field, Real* thl_sec, Real* qw_sec, Real* wthl_sec, Real* w_sec, Real* wqw_sec, Real* qwthl_sec, Real* w3, Real* pres, Real* zt_grid, Real* zi_grid, Real* shoc_cldfrac, Real* shoc_ql, Real* wqls, Real* wthv_sec, Real* shoc_ql2) @@ -2416,7 +1887,7 @@ void shoc_assumed_pdf_f(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* qw, wqw_sec, qwthl_sec, w3, w_field, pres, zt_grid, zi_grid, shoc_cldfrac, shoc_ql, wqls, wthv_sec, shoc_ql2}; // Sync to device - ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d, true); + ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d); // Inputs/Outputs view_2d @@ -2477,9 +1948,9 @@ void shoc_assumed_pdf_f(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* qw, // Sync back to host std::vector out_views = {shoc_cldfrac_d, shoc_ql_d, wqls_d, wthv_sec_d, shoc_ql2_d}; - ekat::device_to_host({shoc_cldfrac, shoc_ql, wqls, wthv_sec, shoc_ql2}, shcol, nlev, out_views, true); + ekat::device_to_host({shoc_cldfrac, shoc_ql, wqls, wthv_sec, shoc_ql2}, shcol, nlev, out_views); } -void compute_shr_prod_f(Int nlevi, Int nlev, Int shcol, Real* dz_zi, Real* u_wind, Real* v_wind, Real* sterm) +void compute_shr_prod_host(Int nlevi, Int nlev, Int shcol, Real* dz_zi, Real* u_wind, Real* v_wind, Real* sterm) { using SHF = Functions; @@ -2497,7 +1968,7 @@ void compute_shr_prod_f(Int nlevi, Int nlev, Int shcol, Real* dz_zi, Real* u_win std::vector ptr_array = {dz_zi, u_wind, v_wind, sterm}; // Sync to device - ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d, true); + ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d); view_2d //input @@ -2525,10 +1996,10 @@ void compute_shr_prod_f(Int nlevi, Int nlev, Int shcol, Real* dz_zi, Real* u_win // Sync back to host std::vector inout_views = {sterm_d}; - ekat::device_to_host({sterm}, shcol, nlevi, inout_views, true); + ekat::device_to_host({sterm}, shcol, nlevi, inout_views); } -void compute_tmpi_f(Int nlevi, Int shcol, Real dtime, Real *rho_zi, Real *dz_zi, Real *tmpi) +void compute_tmpi_host(Int nlevi, Int shcol, Real dtime, Real *rho_zi, Real *dz_zi, Real *tmpi) { using SHF = Functions; @@ -2542,7 +2013,7 @@ void compute_tmpi_f(Int nlevi, Int shcol, Real dtime, Real *rho_zi, Real *dz_zi, // Sync to device std::vector temp_d(num_arrays); - ekat::host_to_device({rho_zi, dz_zi, tmpi}, shcol, nlevi, temp_d, true); + ekat::host_to_device({rho_zi, dz_zi, tmpi}, shcol, nlevi, temp_d); // Inputs/Outputs view_2d @@ -2564,10 +2035,10 @@ void compute_tmpi_f(Int nlevi, Int shcol, Real dtime, Real *rho_zi, Real *dz_zi, // Sync back to host std::vector inout_views = {tmpi_d}; - ekat::device_to_host({tmpi}, shcol, nlevi, inout_views, true); + ekat::device_to_host({tmpi}, shcol, nlevi, inout_views); } -void integ_column_stability_f(Int nlev, Int shcol, Real *dz_zt, +void integ_column_stability_host(Int nlev, Int shcol, Real *dz_zt, Real *pres, Real* brunt, Real *brunt_int) { using SHF = Functions; @@ -2584,7 +2055,7 @@ void integ_column_stability_f(Int nlev, Int shcol, Real *dz_zt, // Sync to device std::vector temp_d(num_arrays); - ekat::host_to_device({dz_zt, pres, brunt}, shcol, nlev, temp_d, true); + ekat::host_to_device({dz_zt, pres, brunt}, shcol, nlev, temp_d); // Inputs view_2d @@ -2619,7 +2090,7 @@ void integ_column_stability_f(Int nlev, Int shcol, Real *dz_zt, ScreamDeepCopy::copy_to_host({brunt_int}, shcol, inout_views); } -void isotropic_ts_f(Int nlev, Int shcol, Real* brunt_int, Real* tke, +void isotropic_ts_host(Int nlev, Int shcol, Real* brunt_int, Real* tke, Real* a_diss, Real* brunt, Real* isotropy) { using SHF = Functions; @@ -2640,7 +2111,7 @@ void isotropic_ts_f(Int nlev, Int shcol, Real* brunt_int, Real* tke, // Sync to device ScreamDeepCopy::copy_to_device({brunt_int}, shcol, temp_1d); - ekat::host_to_device(ptr_array, shcol, nlev, temp_2d, true); + ekat::host_to_device(ptr_array, shcol, nlev, temp_2d); //inputs view_1d brunt_int_d(temp_1d[0]); @@ -2668,8 +2139,8 @@ void isotropic_ts_f(Int nlev, Int shcol, Real* brunt_int, Real* tke, const auto isotropy_s = ekat::subview(isotropy_d, i); //output // Hard code these runtime options for F90 - const Real lambda_low = 0.001; - const Real lambda_high = 0.04; + const Real lambda_low = 0.001; + const Real lambda_high = 0.08; const Real lambda_slope = 2.65; const Real lambda_thresh = 0.02; SHF::isotropic_ts(team, nlev, lambda_low, lambda_high, lambda_slope, lambda_thresh, @@ -2678,11 +2149,11 @@ void isotropic_ts_f(Int nlev, Int shcol, Real* brunt_int, Real* tke, // Sync back to host std::vector inout_views = {isotropy_d}; - ekat::device_to_host({isotropy}, shcol, nlev, inout_views, true); + ekat::device_to_host({isotropy}, shcol, nlev, inout_views); } -void dp_inverse_f(Int nlev, Int shcol, Real *rho_zt, Real *dz_zt, Real *rdp_zt) +void dp_inverse_host(Int nlev, Int shcol, Real *rho_zt, Real *dz_zt, Real *rdp_zt) { using SHF = Functions; @@ -2696,7 +2167,7 @@ void dp_inverse_f(Int nlev, Int shcol, Real *rho_zt, Real *dz_zt, Real *rdp_zt) // Sync to device std::vector temp_d(num_arrays); - ekat::host_to_device({rho_zt, dz_zt, rdp_zt}, shcol, nlev, temp_d, true); + ekat::host_to_device({rho_zt, dz_zt, rdp_zt}, shcol, nlev, temp_d); // Inputs/Outputs view_2d @@ -2718,10 +2189,10 @@ void dp_inverse_f(Int nlev, Int shcol, Real *rho_zt, Real *dz_zt, Real *rdp_zt) // Sync back to host std::vector inout_views = {rdp_zt_d}; - ekat::device_to_host({rdp_zt}, shcol, nlev, inout_views, true); + ekat::device_to_host({rdp_zt}, shcol, nlev, inout_views); } -int shoc_init_f(Int nlev, Real *pref_mid, Int nbot_shoc, Int ntop_shoc) +int shoc_init_host(Int nlev, Real *pref_mid, Int nbot_shoc, Int ntop_shoc) { using SHF = Functions; using Spack = typename SHF::Spack; @@ -2735,7 +2206,7 @@ int shoc_init_f(Int nlev, Real *pref_mid, Int nbot_shoc, Int ntop_shoc) return SHF::shoc_init(nbot_shoc,ntop_shoc,pref_mid_d); } -Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, Real* host_dx, Real* host_dy, Real* thv, Real* zt_grid, +Int shoc_main_host(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, Real* host_dx, Real* host_dy, Real* thv, Real* zt_grid, Real* zi_grid, Real* pres, Real* presi, Real* pdel, Real* wthl_sfc, Real* wqw_sfc, Real* uw_sfc, Real* vw_sfc, Real* wtracer_sfc, Int num_qtracers, Real* w_field, Real* inv_exner, Real* phis, Real* host_dse, Real* tke, Real* thetal, Real* qw, Real* u_wind, Real* v_wind, Real* qtracers, Real* wthv_sec, Real* tkh, Real* tk, @@ -2783,14 +2254,14 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, std::vector ptr_array_2d = {zt_grid, zi_grid, pres, presi, pdel, thv, w_field, wtracer_sfc, inv_exner, host_dse, tke, thetal, qw, u_wind, v_wind, - wthv_sec, tk, shoc_cldfrac, shoc_ql, shoc_ql2, + wthv_sec, tk, shoc_cldfrac, shoc_ql, shoc_ql2, tkh, shoc_mix, w_sec, thl_sec, qw_sec, qwthl_sec, wthl_sec, wqw_sec, wtke_sec, uw_sec, vw_sec, w3, wqls_sec, brunt, isotropy}; ScreamDeepCopy::copy_to_device(ptr_array_1d, shcol, temp_1d_d); - ekat::host_to_device(ptr_array_2d, dim1_2d_sizes, dim2_2d_sizes, temp_2d_d, true); - ekat::host_to_device({qtracers}, shcol, nlev, num_qtracers, temp_3d_d, true); + ekat::host_to_device(ptr_array_2d, dim1_2d_sizes, dim2_2d_sizes, temp_2d_d); + ekat::host_to_device({qtracers}, shcol, nlev, num_qtracers, temp_3d_d); Int index_counter = 0; view_1d @@ -2972,16 +2443,16 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, qw_sec_d, qwthl_sec_d, wthl_sec_d, wqw_sec_d, wtke_sec_d, uw_sec_d, vw_sec_d, w3_d, wqls_sec_d, brunt_d, isotropy_d}; - ekat::device_to_host(ptr_array_2d_out, dim1_2d_out, dim2_2d_out, out_views_2d, true); + ekat::device_to_host(ptr_array_2d_out, dim1_2d_out, dim2_2d_out, out_views_2d); // 3d std::vector out_views_3d = {qtracers_f90_d}; - ekat::device_to_host({qtracers}, shcol, nlev, num_qtracers, out_views_3d, true); + ekat::device_to_host({qtracers}, shcol, nlev, num_qtracers, out_views_3d); return elapsed_microsec; } -void pblintd_height_f(Int shcol, Int nlev, Int npbl, Real* z, Real* u, Real* v, Real* ustar, Real* thv, Real* thv_ref, Real* pblh, Real* rino, bool* check) +void pblintd_height_host(Int shcol, Int nlev, Int npbl, Real* z, Real* u, Real* v, Real* ustar, Real* thv, Real* thv_ref, Real* pblh, Real* rino, bool* check) { using SHOC = Functions; using Spack = typename SHOC::Spack; @@ -2993,7 +2464,7 @@ void pblintd_height_f(Int shcol, Int nlev, Int npbl, Real* z, Real* u, Real* v, using MemberType = typename SHOC::MemberType; std::vector views_2d(5); - ekat::host_to_device({z, u, v, thv, rino}, shcol, nlev, views_2d, true); + ekat::host_to_device({z, u, v, thv, rino}, shcol, nlev, views_2d); view_2d z_2d (views_2d[0]), u_2d (views_2d[1]), @@ -3034,14 +2505,13 @@ void pblintd_height_f(Int shcol, Int nlev, Int npbl, Real* z, Real* u, Real* v, ScreamDeepCopy::copy_to_host({pblh}, shcol, out_1d_views); std::vector out_2d_views = {rino_2d}; - ekat::device_to_host({rino}, shcol, nlev, out_2d_views, true); + ekat::device_to_host({rino}, shcol, nlev, out_2d_views); std::vector out_bool_1d_views = {check_1d}; ScreamDeepCopy::copy_to_host({check}, shcol, out_bool_1d_views); } -void vd_shoc_decomp_and_solve_f(Int shcol, Int nlev, Int nlevi, Int num_rhs, Real* kv_term, Real* tmpi, Real* rdp_zt, Real dtime, - Real* flux, Real* var) +void vd_shoc_decomp_and_solve_host(Int shcol, Int nlev, Int nlevi, Int num_rhs, Real dtime, Real* kv_term, Real* tmpi, Real* rdp_zt, Real* flux, Real* var) { using SHF = Functions; @@ -3069,8 +2539,8 @@ void vd_shoc_decomp_and_solve_f(Int shcol, Int nlev, Int nlevi, Int num_rhs, Rea // Sync to device ScreamDeepCopy::copy_to_device({flux}, shcol, temp_1d_d); - ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_2d_d, true); - ekat::host_to_device({var}, shcol, nlev, num_rhs, temp_3d_d, true); + ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_2d_d); + ekat::host_to_device({var}, shcol, nlev, num_rhs, temp_3d_d); view_1d flux_d(temp_1d_d[0]); @@ -3109,10 +2579,10 @@ void vd_shoc_decomp_and_solve_f(Int shcol, Int nlev, Int nlevi, Int num_rhs, Rea // Sync back to host std::vector inout_views = {var_d}; - ekat::device_to_host({var}, shcol, nlev, num_rhs, inout_views, true); + ekat::device_to_host({var}, shcol, nlev, num_rhs, inout_views); } -void shoc_grid_f(Int shcol, Int nlev, Int nlevi, Real* zt_grid, Real* zi_grid, Real* pdel, Real* dz_zt, Real* dz_zi, Real* rho_zt) +void shoc_grid_host(Int shcol, Int nlev, Int nlevi, Real* zt_grid, Real* zi_grid, Real* pdel, Real* dz_zt, Real* dz_zi, Real* rho_zt) { using SHF = Functions; @@ -3131,7 +2601,7 @@ void shoc_grid_f(Int shcol, Int nlev, Int nlevi, Real* zt_grid, Real* zi_grid, R dz_zt, dz_zi, rho_zt}; // Sync to device - ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_2d_d, true); + ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_2d_d); view_2d zt_grid_d(temp_2d_d[0]), @@ -3158,10 +2628,10 @@ void shoc_grid_f(Int shcol, Int nlev, Int nlevi, Real* zt_grid, Real* zi_grid, R // Sync back to host std::vector inout_views = {dz_zt_d, dz_zi_d, rho_zt_d}; - ekat::device_to_host({dz_zt, dz_zi, rho_zt}, {shcol, shcol, shcol}, {nlev, nlevi, nlev}, inout_views, true); + ekat::device_to_host({dz_zt, dz_zi, rho_zt}, {shcol, shcol, shcol}, {nlev, nlevi, nlev}, inout_views); } -void eddy_diffusivities_f(Int nlev, Int shcol, Real* pblh, Real* zt_grid, Real* tabs, Real* shoc_mix, Real* sterm_zt, +void eddy_diffusivities_host(Int nlev, Int shcol, Real* pblh, Real* zt_grid, Real* tabs, Real* shoc_mix, Real* sterm_zt, Real* isotropy, Real* tke, Real* tkh, Real* tk) { using SHF = Functions; @@ -3185,7 +2655,7 @@ void eddy_diffusivities_f(Int nlev, Int shcol, Real* pblh, Real* zt_grid, Real* // Sync to device ScreamDeepCopy::copy_to_device({pblh}, shcol, temp_1d_d); - ekat::host_to_device(ptr_array, shcol, nlev, temp_2d_d, true); + ekat::host_to_device(ptr_array, shcol, nlev, temp_2d_d); view_1d pblh_d(temp_1d_d[0]); @@ -3223,10 +2693,10 @@ void eddy_diffusivities_f(Int nlev, Int shcol, Real* pblh, Real* zt_grid, Real* // Sync back to host std::vector inout_views = {tkh_d, tk_d}; - ekat::device_to_host({tkh, tk}, shcol, nlev, inout_views, true); + ekat::device_to_host({tkh, tk}, shcol, nlev, inout_views); } -void pblintd_surf_temp_f(Int shcol, Int nlev, Int nlevi, Real* z, Real* ustar, Real* obklen, Real* kbfs, Real* thv, Real* tlv, Real* pblh, bool* check, Real* rino) +void pblintd_surf_temp_host(Int shcol, Int nlev, Int nlevi, Real* z, Real* ustar, Real* obklen, Real* kbfs, Real* thv, Real* tlv, Real* pblh, bool* check, Real* rino) { using SHOC = Functions; using Spack = typename SHOC::Spack; @@ -3236,7 +2706,7 @@ void pblintd_surf_temp_f(Int shcol, Int nlev, Int nlevi, Real* z, Real* ustar, R using view_2d = typename SHOC::view_2d; std::vector views_2d(3); - ekat::host_to_device({z, thv, rino}, shcol, nlev, views_2d, true); + ekat::host_to_device({z, thv, rino}, shcol, nlev, views_2d); view_2d z_2d (views_2d[0]), thv_2d (views_2d[1]), rino_2d(views_2d[2]); @@ -3277,13 +2747,13 @@ void pblintd_surf_temp_f(Int shcol, Int nlev, Int nlevi, Real* z, Real* ustar, R ScreamDeepCopy::copy_to_host({pblh, tlv}, shcol, out_1d_views); std::vector out_2d_views = {rino_2d}; - ekat::device_to_host({rino}, shcol, nlev, out_2d_views, true); + ekat::device_to_host({rino}, shcol, nlev, out_2d_views); std::vector out_bool_1d_views = {check_1d}; ScreamDeepCopy::copy_to_host({check}, shcol, out_bool_1d_views); } -void pblintd_check_pblh_f(Int shcol, Int nlev, Int nlevi, Int npbl, Real* z, Real* ustar, bool* check, Real* pblh) +void pblintd_check_pblh_host(Int shcol, Int nlev, Int nlevi, Int npbl, Real* z, Real* ustar, bool* check, Real* pblh) { using SHOC = Functions; using Spack = typename SHOC::Spack; @@ -3293,7 +2763,7 @@ void pblintd_check_pblh_f(Int shcol, Int nlev, Int nlevi, Int npbl, Real* z, Rea using view_2d = typename SHOC::view_2d; std::vector views_2d(1); - ekat::host_to_device({z}, shcol, nlev, views_2d, true); + ekat::host_to_device({z}, shcol, nlev, views_2d); view_2d z_2d (views_2d[0]); std::vector views_1d(2); @@ -3319,7 +2789,7 @@ void pblintd_check_pblh_f(Int shcol, Int nlev, Int nlevi, Int npbl, Real* z, Rea ScreamDeepCopy::copy_to_host({pblh}, shcol, out_1d_views); } -void pblintd_f(Int shcol, Int nlev, Int nlevi, Int npbl, Real* z, Real* zi, Real* thl, Real* ql, Real* q, Real* u, Real* v, Real* ustar, Real* obklen, Real* kbfs, Real* cldn, Real* pblh) +void pblintd_host(Int shcol, Int nlev, Int nlevi, Int npbl, Real* z, Real* zi, Real* thl, Real* ql, Real* q, Real* u, Real* v, Real* ustar, Real* obklen, Real* kbfs, Real* cldn, Real* pblh) { using SHF = Functions; @@ -3345,7 +2815,7 @@ void pblintd_f(Int shcol, Int nlev, Int nlevi, Int npbl, Real* z, Real* zi, Real // Sync to device ScreamDeepCopy::copy_to_device({ustar, obklen, kbfs}, shcol, temp_1d_d); - ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_2d_d, true); + ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_2d_d); view_1d ustar_d(temp_1d_d[0]), @@ -3399,7 +2869,7 @@ void pblintd_f(Int shcol, Int nlev, Int nlevi, Int npbl, Real* z, Real* zi, Real ScreamDeepCopy::copy_to_host({pblh}, shcol, out_views); } -void shoc_tke_f(Int shcol, Int nlev, Int nlevi, Real dtime, Real* wthv_sec, Real* shoc_mix, Real* dz_zi, Real* dz_zt, Real* pres, +void shoc_tke_host(Int shcol, Int nlev, Int nlevi, Real dtime, Real* wthv_sec, Real* shoc_mix, Real* dz_zi, Real* dz_zt, Real* pres, Real* tabs, Real* u_wind, Real* v_wind, Real* brunt, Real* zt_grid, Real* zi_grid, Real* pblh, Real* tke, Real* tk, Real* tkh, Real* isotropy) { @@ -3427,7 +2897,7 @@ void shoc_tke_f(Int shcol, Int nlev, Int nlevi, Real dtime, Real* wthv_sec, Real // Sync to device ScreamDeepCopy::copy_to_device({pblh}, shcol, temp_1d_d); - ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_2d_d, true); + ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_2d_d); view_1d pblh_d(temp_1d_d[0]); @@ -3479,8 +2949,8 @@ void shoc_tke_f(Int shcol, Int nlev, Int nlevi, Real dtime, Real* wthv_sec, Real const auto isotropy_s = ekat::subview(isotropy_d, i); // Hardcode for F90 testing - const Real lambda_low = 0.001; - const Real lambda_high = 0.04; + const Real lambda_low = 0.001; + const Real lambda_high = 0.08; const Real lambda_slope = 2.65; const Real lambda_thresh = 0.02; const Real Ckh = 0.1; @@ -3496,10 +2966,10 @@ void shoc_tke_f(Int shcol, Int nlev, Int nlevi, Real dtime, Real* wthv_sec, Real // Sync back to host std::vector inout_views = {tke_d, tk_d, tkh_d, isotropy_d}; - ekat::device_to_host({tke, tk, tkh, isotropy}, shcol, nlev, inout_views, true); + ekat::device_to_host({tke, tk, tkh, isotropy}, shcol, nlev, inout_views); } -void compute_shoc_temperature_f(Int shcol, Int nlev, Real *thetal, Real *ql, Real *inv_exner, Real* tabs) +void compute_shoc_temperature_host(Int shcol, Int nlev, Real *thetal, Real *ql, Real *inv_exner, Real* tabs) { using SHF = Functions; @@ -3513,7 +2983,7 @@ void compute_shoc_temperature_f(Int shcol, Int nlev, Real *thetal, Real *ql, Rea // Sync to device std::vector temp_d(num_arrays); - ekat::host_to_device({thetal, ql, inv_exner, tabs}, shcol, nlev, temp_d, true); + ekat::host_to_device({thetal, ql, inv_exner, tabs}, shcol, nlev, temp_d); // Inputs/Outputs view_2d @@ -3537,7 +3007,280 @@ void compute_shoc_temperature_f(Int shcol, Int nlev, Real *thetal, Real *ql, Rea // Sync back to host std::vector out_views = {tabs_d}; - ekat::device_to_host({tabs}, shcol, nlev, out_views, true); + ekat::device_to_host({tabs}, shcol, nlev, out_views); +} + +void shoc_assumed_pdf_tilde_to_real_host(Real w_first, Real sqrtw2, Real* w1) +{ + using SHF = Functions; + + using Spack = typename SHF::Spack; + using view_1d = typename SHF::view_1d; + + view_1d t_d("t_d", 1); + const auto t_h = Kokkos::create_mirror_view(t_d); + + Real local_w1(*w1); + Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + Spack sqrtw2_(sqrtw2), w1_(local_w1), w_first_(w_first); + SHF::shoc_assumed_pdf_tilde_to_real(w_first_, sqrtw2_, w1_); + t_d(0) = w1_[0]; + }); + Kokkos::deep_copy(t_h, t_d); + *w1 = t_h(0); +} + +void shoc_assumed_pdf_vv_parameters_host(Real w_first, Real w_sec, Real w3var, Real w_tol_sqd, Real* skew_w, Real* w1_1, Real* w1_2, Real* w2_1, Real* w2_2, Real* a) +{ + using SHF = Functions; + + using Spack = typename SHF::Spack; + using view_1d = typename SHF::view_1d; + + view_1d t_d("t_d", 6); + const auto t_h = Kokkos::create_mirror_view(t_d); + + Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + Spack w3var_(w3var), w_first_(w_first), w_sec_(w_sec), a_, skew_w_, w1_1_, w1_2_, w2_1_, w2_2_; + SHF::shoc_assumed_pdf_vv_parameters(w_first_, w_sec_, w3var_, w_tol_sqd, skew_w_, w1_1_, w1_2_, w2_1_, w2_2_, a_); + t_d(0) = a_[0]; + t_d(1) = skew_w_[0]; + t_d(2) = w1_1_[0]; + t_d(3) = w1_2_[0]; + t_d(4) = w2_1_[0]; + t_d(5) = w2_2_[0]; + }); + Kokkos::deep_copy(t_h, t_d); + *a = t_h(0); + *skew_w = t_h(1); + *w1_1 = t_h(2); + *w1_2 = t_h(3); + *w2_1 = t_h(4); + *w2_2 = t_h(5); +} + +void shoc_assumed_pdf_thl_parameters_host(Real wthlsec, Real sqrtw2, Real sqrtthl, Real thlsec, Real thl_first, Real w1_1, Real w1_2, Real skew_w, Real a, Real thl_tol, Real w_thresh, Real* thl1_1, Real* thl1_2, Real* thl2_1, Real* thl2_2, Real* sqrtthl2_1, Real* sqrtthl2_2) +{ + using SHF = Functions; + + using Spack = typename SHF::Spack; + using view_1d = typename SHF::view_1d; + + view_1d t_d("t_d", 6); + const auto t_h = Kokkos::create_mirror_view(t_d); + + Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + Spack a_(a), skew_w_(skew_w), sqrtthl_(sqrtthl), sqrtw2_(sqrtw2), thl_first_(thl_first), thlsec_(thlsec), w1_1_(w1_1), w1_2_(w1_2), wthlsec_(wthlsec), sqrtthl2_1_, sqrtthl2_2_, thl1_1_, thl1_2_, thl2_1_, thl2_2_; + SHF::shoc_assumed_pdf_thl_parameters(wthlsec_, sqrtw2_, sqrtthl_, thlsec_, thl_first_, w1_1_, w1_2_, skew_w_, a_, thl_tol, w_thresh, thl1_1_, thl1_2_, thl2_1_, thl2_2_, sqrtthl2_1_, sqrtthl2_2_); + t_d(0) = sqrtthl2_1_[0]; + t_d(1) = sqrtthl2_2_[0]; + t_d(2) = thl1_1_[0]; + t_d(3) = thl1_2_[0]; + t_d(4) = thl2_1_[0]; + t_d(5) = thl2_2_[0]; + }); + Kokkos::deep_copy(t_h, t_d); + *sqrtthl2_1 = t_h(0); + *sqrtthl2_2 = t_h(1); + *thl1_1 = t_h(2); + *thl1_2 = t_h(3); + *thl2_1 = t_h(4); + *thl2_2 = t_h(5); +} + +void shoc_assumed_pdf_qw_parameters_host(Real wqwsec, Real sqrtw2, Real skew_w, Real sqrtqt, Real qwsec, Real w1_2, Real w1_1, Real qw_first, Real a, Real rt_tol, Real w_thresh, Real* qw1_1, Real* qw1_2, Real* qw2_1, Real* qw2_2, Real* sqrtqw2_1, Real* sqrtqw2_2) +{ + using SHF = Functions; + + using Spack = typename SHF::Spack; + using view_1d = typename SHF::view_1d; + + view_1d t_d("t_d", 6); + const auto t_h = Kokkos::create_mirror_view(t_d); + + Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + Spack a_(a), qw_first_(qw_first), qwsec_(qwsec), skew_w_(skew_w), sqrtqt_(sqrtqt), sqrtw2_(sqrtw2), w1_1_(w1_1), w1_2_(w1_2), wqwsec_(wqwsec), qw1_1_, qw1_2_, qw2_1_, qw2_2_, sqrtqw2_1_, sqrtqw2_2_; + SHF::shoc_assumed_pdf_qw_parameters(wqwsec_, sqrtw2_, skew_w_, sqrtqt_, qwsec_, w1_2_, w1_1_, qw_first_, a_, rt_tol, w_thresh, qw1_1_, qw1_2_, qw2_1_, qw2_2_, sqrtqw2_1_, sqrtqw2_2_); + t_d(0) = qw1_1_[0]; + t_d(1) = qw1_2_[0]; + t_d(2) = qw2_1_[0]; + t_d(3) = qw2_2_[0]; + t_d(4) = sqrtqw2_1_[0]; + t_d(5) = sqrtqw2_2_[0]; + }); + Kokkos::deep_copy(t_h, t_d); + *qw1_1 = t_h(0); + *qw1_2 = t_h(1); + *qw2_1 = t_h(2); + *qw2_2 = t_h(3); + *sqrtqw2_1 = t_h(4); + *sqrtqw2_2 = t_h(5); +} + +void shoc_assumed_pdf_inplume_correlations_host(Real sqrtqw2_1, Real sqrtthl2_1, Real a, Real sqrtqw2_2, Real sqrtthl2_2, Real qwthlsec, Real qw1_1, Real qw_first, Real thl1_1, Real thl_first, Real qw1_2, Real thl1_2, Real* r_qwthl_1) +{ + using SHF = Functions; + + using Spack = typename SHF::Spack; + using view_1d = typename SHF::view_1d; + + view_1d t_d("t_d", 1); + const auto t_h = Kokkos::create_mirror_view(t_d); + + Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + Spack a_(a), qw1_1_(qw1_1), qw1_2_(qw1_2), qw_first_(qw_first), qwthlsec_(qwthlsec), sqrtqw2_1_(sqrtqw2_1), sqrtqw2_2_(sqrtqw2_2), sqrtthl2_1_(sqrtthl2_1), sqrtthl2_2_(sqrtthl2_2), thl1_1_(thl1_1), thl1_2_(thl1_2), thl_first_(thl_first), r_qwthl_1_; + SHF::shoc_assumed_pdf_inplume_correlations(sqrtqw2_1_, sqrtthl2_1_, a_, sqrtqw2_2_, sqrtthl2_2_, qwthlsec_, qw1_1_, qw_first_, thl1_1_, thl_first_, qw1_2_, thl1_2_, r_qwthl_1_); + t_d(0) = r_qwthl_1_[0]; + }); + Kokkos::deep_copy(t_h, t_d); + *r_qwthl_1 = t_h(0); +} + +void shoc_assumed_pdf_compute_temperature_host(Real thl1, Real pval, Real* tl1) +{ + using SHF = Functions; + + using Spack = typename SHF::Spack; + using view_1d = typename SHF::view_1d; + + view_1d t_d("t_d", 1); + const auto t_h = Kokkos::create_mirror_view(t_d); + + Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + Spack pval_(pval), thl1_(thl1), tl1_; + SHF::shoc_assumed_pdf_compute_temperature(thl1_, pval_, tl1_); + t_d(0) = tl1_[0]; + }); + Kokkos::deep_copy(t_h, t_d); + *tl1 = t_h(0); +} + +void shoc_assumed_pdf_compute_qs_host(Real tl1_1, Real tl1_2, Real pval, Real* qs1, Real* beta1, Real* qs2, Real* beta2) +{ + using SHF = Functions; + + using Spack = typename SHF::Spack; + using Smask = typename SHF::Smask; + using view_1d = typename SHF::view_1d; + + view_1d t_d("t_d", 4); + const auto t_h = Kokkos::create_mirror_view(t_d); + + Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + Spack pval_(pval), tl1_1_(tl1_1), tl1_2_(tl1_2), beta1_, beta2_, qs1_, qs2_; + Smask active_entries(true); + SHF::shoc_assumed_pdf_compute_qs(tl1_1_, tl1_2_, pval_, active_entries, qs1_, beta1_, qs2_, beta2_); + t_d(0) = beta1_[0]; + t_d(1) = beta2_[0]; + t_d(2) = qs1_[0]; + t_d(3) = qs2_[0]; + }); + Kokkos::deep_copy(t_h, t_d); + *beta1 = t_h(0); + *beta2 = t_h(1); + *qs1 = t_h(2); + *qs2 = t_h(3); +} + +void shoc_assumed_pdf_compute_s_host(Real qw1, Real qs1, Real beta, Real pval, Real thl2, Real qw2, Real sqrtthl2, Real sqrtqw2, Real r_qwthl, Real* s, Real* std_s, Real* qn, Real* c) +{ + using SHF = Functions; + + using Spack = typename SHF::Spack; + using view_1d = typename SHF::view_1d; + + view_1d t_d("t_d", 4); + const auto t_h = Kokkos::create_mirror_view(t_d); + + Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + Spack beta_(beta), pval_(pval), qs1_(qs1), qw1_(qw1), qw2_(qw2), r_qwthl_(r_qwthl), sqrtqw2_(sqrtqw2), sqrtthl2_(sqrtthl2), thl2_(thl2), c_, qn_, s_, std_s_; + SHF::shoc_assumed_pdf_compute_s(qw1_, qs1_, beta_, pval_, thl2_, qw2_, sqrtthl2_, sqrtqw2_, r_qwthl_, s_, std_s_, qn_, c_); + t_d(0) = c_[0]; + t_d(1) = qn_[0]; + t_d(2) = s_[0]; + t_d(3) = std_s_[0]; + }); + Kokkos::deep_copy(t_h, t_d); + *c = t_h(0); + *qn = t_h(1); + *s = t_h(2); + *std_s = t_h(3); +} + +void shoc_assumed_pdf_compute_sgs_liquid_host(Real a, Real ql1, Real ql2, Real* shoc_ql) +{ + using SHF = Functions; + + using Spack = typename SHF::Spack; + using view_1d = typename SHF::view_1d; + + view_1d t_d("t_d", 1); + const auto t_h = Kokkos::create_mirror_view(t_d); + + Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + Spack a_(a), ql1_(ql1), ql2_(ql2), shoc_ql_; + SHF::shoc_assumed_pdf_compute_sgs_liquid(a_, ql1_, ql2_, shoc_ql_); + t_d(0) = shoc_ql_[0]; + }); + Kokkos::deep_copy(t_h, t_d); + *shoc_ql = t_h(0); +} + +void shoc_assumed_pdf_compute_cloud_liquid_variance_host(Real a, Real s1, Real ql1, Real c1, Real std_s1, Real s2, Real ql2, Real c2, Real std_s2, Real shoc_ql, Real* shoc_ql2) +{ + using SHF = Functions; + + using Spack = typename SHF::Spack; + using view_1d = typename SHF::view_1d; + + view_1d t_d("t_d", 1); + const auto t_h = Kokkos::create_mirror_view(t_d); + + Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + Spack a_(a), c1_(c1), c2_(c2), ql1_(ql1), ql2_(ql2), s1_(s1), s2_(s2), shoc_ql_(shoc_ql), std_s1_(std_s1), std_s2_(std_s2), shoc_ql2_; + SHF::shoc_assumed_pdf_compute_cloud_liquid_variance(a_, s1_, ql1_, c1_, std_s1_, s2_, ql2_, c2_, std_s2_, shoc_ql_, shoc_ql2_); + t_d(0) = shoc_ql2_[0]; + }); + Kokkos::deep_copy(t_h, t_d); + *shoc_ql2 = t_h(0); +} + +void shoc_assumed_pdf_compute_liquid_water_flux_host(Real a, Real w1_1, Real w_first, Real ql1, Real w1_2, Real ql2, Real* wqls) +{ + using SHF = Functions; + + using Spack = typename SHF::Spack; + using view_1d = typename SHF::view_1d; + + view_1d t_d("t_d", 1); + const auto t_h = Kokkos::create_mirror_view(t_d); + + Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + Spack a_(a), ql1_(ql1), ql2_(ql2), w1_1_(w1_1), w1_2_(w1_2), w_first_(w_first), wqls_; + SHF::shoc_assumed_pdf_compute_liquid_water_flux(a_, w1_1_, w_first_, ql1_, w1_2_, ql2_, wqls_); + t_d(0) = wqls_[0]; + }); + Kokkos::deep_copy(t_h, t_d); + *wqls = t_h(0); +} + +void shoc_assumed_pdf_compute_buoyancy_flux_host(Real wthlsec, Real wqwsec, Real pval, Real wqls, Real* wthv_sec) +{ + using PF = Functions; + + using Spack = typename PF::Spack; + using view_1d = typename PF::view_1d; + + view_1d t_d("t_d", 1); + const auto t_h = Kokkos::create_mirror_view(t_d); + + Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + Spack pval_(pval), wqls_(wqls), wqwsec_(wqwsec), wthlsec_(wthlsec), wthv_sec_; + PF::shoc_assumed_pdf_compute_buoyancy_flux(wthlsec_, wqwsec_, pval_, wqls_, wthv_sec_); + t_d(0) = wthv_sec_[0]; + }); + Kokkos::deep_copy(t_h, t_d); + *wthv_sec = t_h(0); } } // namespace shoc diff --git a/components/eamxx/src/physics/shoc/shoc_functions_f90.hpp b/components/eamxx/src/physics/shoc/tests/infra/shoc_test_data.hpp similarity index 80% rename from components/eamxx/src/physics/shoc/shoc_functions_f90.hpp rename to components/eamxx/src/physics/shoc/tests/infra/shoc_test_data.hpp index bc61efb168e6..2eb4236b4ed0 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions_f90.hpp +++ b/components/eamxx/src/physics/shoc/tests/infra/shoc_test_data.hpp @@ -137,51 +137,6 @@ struct ShocEnergyIntegralsData : public PhysicsTestData { PTD_STD_DEF(ShocEnergyIntegralsData, 2, shcol, nlev); }; -struct ShocEnergyTotalFixerData : public ShocTestGridDataBase { - // Inputs - Int shcol, nlev, nlevi, nadv; - Real dtime; - Real *se_b, *ke_b, *wv_b, *wl_b, *se_a, *ke_a, *wv_a, *wl_a, *wthl_sfc, *wqw_sfc, *rho_zt, *pint; - - // Outputs - Real *te_a, *te_b; - - ShocEnergyTotalFixerData(Int shcol_, Int nlev_, Int nlevi_, Real dtime_, Int nadv_) : - ShocTestGridDataBase({{ shcol_, nlev_ }, { shcol_, nlevi_ }, { shcol_ }}, {{ &zt_grid, &rho_zt }, { &zi_grid, &pint }, { &se_b, &ke_b, &wv_b, &wl_b, &se_a, &ke_a, &wv_a, &wl_a, &wthl_sfc, &wqw_sfc, &te_a, &te_b }}), shcol(shcol_), nlev(nlev_), nlevi(nlevi_), nadv(nadv_), dtime(dtime_) {} - - PTD_STD_DEF(ShocEnergyTotalFixerData, 5, shcol, nlev, nlevi, dtime, nadv); -}; - -struct ShocEnergyThresholdFixerData : public PhysicsTestData { - // Inputs - Int shcol, nlev, nlevi; - Real *pint, *tke, *te_a, *te_b; - - // Outputs - Real *se_dis; - Int *shoctop; - - ShocEnergyThresholdFixerData(Int shcol_, Int nlev_, Int nlevi_) : - PhysicsTestData({{ shcol_, nlevi_ }, { shcol_, nlev_ }, { shcol_ }, { shcol_ }}, {{ &pint }, { &tke }, { &te_a, &te_b, &se_dis }}, {{ &shoctop }}), shcol(shcol_), nlev(nlev_), nlevi(nlevi_) {} - - PTD_STD_DEF(ShocEnergyThresholdFixerData, 3, shcol, nlev, nlevi); -}; - -struct ShocEnergyDseFixerData : public PhysicsTestData { - // Inputs - Int shcol, nlev; - Real *se_dis; - Int *shoctop; - - // Inputs/Outputs - Real *host_dse; - - ShocEnergyDseFixerData(Int shcol_, Int nlev_) : - PhysicsTestData({{ shcol_ }, { shcol_, nlev_ }, { shcol_ }}, {{ &se_dis }, { &host_dse }}, {{ &shoctop }}), shcol(shcol_), nlev(nlev_) {} - - PTD_STD_DEF(ShocEnergyDseFixerData, 2, shcol, nlev); -}; - struct CalcShocVertfluxData : public PhysicsTestData { // Inputs Int shcol, nlev, nlevi; @@ -240,49 +195,6 @@ struct DpInverseData : public PhysicsTestData { PTD_STD_DEF(DpInverseData, 2, shcol, nlev); }; -struct SfcFluxesData : public PhysicsTestData { - // Inputs - Int shcol, num_tracer; - Real dtime; - Real *rho_zi_sfc, *rdp_zt_sfc, *wthl_sfc, *wqw_sfc, *wtke_sfc, *wtracer_sfc; - - // Inputs/Outputs - Real *thetal, *qw, *tke, *wtracer; - - SfcFluxesData(Int shcol_, Int num_tracer_, Real dtime_) : - PhysicsTestData({{ shcol_ }, { shcol_, num_tracer_ }}, {{ &rho_zi_sfc, &rdp_zt_sfc, &wthl_sfc, &wqw_sfc, &wtke_sfc, &thetal, &qw, &tke }, { &wtracer_sfc, &wtracer }}), shcol(shcol_), num_tracer(num_tracer_), dtime(dtime_) {} - - PTD_STD_DEF(SfcFluxesData, 3, shcol, num_tracer, dtime); -}; - -struct ImpliSrfStressTermData : public PhysicsTestData { - // Inputs - Int shcol; - Real *rho_zi_sfc, *uw_sfc, *vw_sfc, *u_wind_sfc, *v_wind_sfc; - - // Outputs - Real *ksrf; - - ImpliSrfStressTermData(Int shcol_) : - PhysicsTestData({{ shcol_ }}, {{ &rho_zi_sfc, &uw_sfc, &vw_sfc, &u_wind_sfc, &v_wind_sfc, &ksrf }}), shcol(shcol_) {} - - PTD_STD_DEF(ImpliSrfStressTermData, 1, shcol); -}; - -struct TkeSrfFluxTermData : public PhysicsTestData { - // Inputs - Int shcol; - Real *uw_sfc, *vw_sfc; - - // Outputs - Real *wtke_sfc; - - TkeSrfFluxTermData(Int shcol_) : - PhysicsTestData({{ shcol_ }}, {{ &uw_sfc, &vw_sfc, &wtke_sfc }}), shcol(shcol_) {} - - PTD_STD_DEF(TkeSrfFluxTermData, 1, shcol); -}; - struct IntegColumnStabilityData : public PhysicsTestData { // Inputs Int shcol, nlev; @@ -476,54 +388,6 @@ struct CheckLengthScaleShocLengthData : public PhysicsTestData { PTD_STD_DEF(CheckLengthScaleShocLengthData, 2, shcol, nlev); }; -struct FtermsInputForDiagThirdShocMomentData { - // Inputs - Real dz_zi, dz_zt, dz_zt_kc, isotropy_zi, brunt_zi, thetal_zi; - - // Outputs - Real thedz, thedz2, iso, isosqrd, buoy_sgs2, bet2; -}; - -struct AaTermsDiagThirdShocMomentData { - // Inputs - Real omega0, omega1, omega2, x0, x1, y0, y1; - - // Outputs - Real aa0, aa1; -}; - -struct F0ToF5DiagThirdShocMomentData { - // Inputs - Real thedz, thedz2, bet2, iso, isosqrd, wthl_sec, wthl_sec_kc, wthl_sec_kb, thl_sec_kc, thl_sec_kb, w_sec, w_sec_kc, w_sec_zi, tke, tke_kc; - - // Outputs - Real f0, f1, f2, f3, f4, f5; -}; - -struct OmegaTermsDiagThirdShocMomentData { - // Inputs - Real buoy_sgs2, f3, f4; - - // Outputs - Real omega0, omega1, omega2; -}; - -struct XYTermsDiagThirdShocMomentData { - // Inputs - Real buoy_sgs2, f0, f1, f2; - - // Outputs - Real x0, y0, x1, y1; -}; - -struct W3DiagThirdShocMomentData { - // Inputs - Real aa0, aa1, x0, x1, f5; - - // Outputs - Real w3; -}; - struct ClippingDiagThirdShocMomentsData : public PhysicsTestData { // Inputs Int shcol, nlevi; @@ -619,7 +483,7 @@ struct ShocAssumedPdfTildeToRealData { struct ShocAssumedPdfVvParametersData { // Inputs - Real w_first, w_sec, w3var; + Real w_first, w_sec, w3var, w_tol_sqd; // Outputs Real skew_w, w1_1, w1_2, w2_1, w2_2, a; @@ -627,8 +491,7 @@ struct ShocAssumedPdfVvParametersData { struct ShocAssumedPdfThlParametersData { // Inputs - Real wthlsec, sqrtw2, sqrtthl, thlsec, thl_first, w1_1, w1_2, skew_w, a; - bool dothetal_skew; + Real wthlsec, sqrtw2, sqrtthl, thlsec, thl_first, w1_1, w1_2, skew_w, a, thl_tol, w_thresh; // Outputs Real thl1_1, thl1_2, thl2_1, thl2_2, sqrtthl2_1, sqrtthl2_2; @@ -636,7 +499,7 @@ struct ShocAssumedPdfThlParametersData { struct ShocAssumedPdfQwParametersData { // Inputs - Real wqwsec, sqrtw2, skew_w, sqrtqt, qwsec, w1_2, w1_1, qw_first, a; + Real wqwsec, sqrtw2, skew_w, sqrtqt, qwsec, w1_2, w1_1, qw_first, a, rt_tol, w_thresh; // Outputs Real qw1_1, qw1_2, qw2_1, qw2_2, sqrtqw2_1, sqrtqw2_2; @@ -652,7 +515,7 @@ struct ShocAssumedPdfInplumeCorrelationsData { struct ShocAssumedPdfComputeTemperatureData { // Inputs - Real thl1, basepres, pval; + Real thl1, pval; // Outputs Real tl1; @@ -700,7 +563,7 @@ struct ShocAssumedPdfComputeLiquidWaterFluxData { struct ShocAssumedPdfComputeBuoyancyFluxData { // Inputs - Real wthlsec, epsterm, wqwsec, pval, wqls; + Real wthlsec, wqwsec, pval, wqls; // Outputs Real wthv_sec; @@ -997,12 +860,11 @@ struct VdShocDecompandSolveData : public PhysicsTestData { Real *kv_term, *tmpi, *rdp_zt, *flux; // Inputs/Outputs - Real *du, *dl, *d; - Real *var, *rhs; + Real *var; VdShocDecompandSolveData(Int shcol_, Int nlev_, Int nlevi_, Real dtime_, Int n_rhs_) : PhysicsTestData({{shcol_}, {shcol_, nlev_}, {shcol_, nlevi_}, {shcol_, nlev_, n_rhs_}}, - {{&flux}, {&rdp_zt, &du, &dl, &d, &rhs}, {&kv_term, &tmpi}, {&var } }, {}), + {{&flux}, {&rdp_zt}, {&kv_term, &tmpi}, {&var } }, {}), shcol(shcol_), nlev(nlev_), nlevi(nlevi_), n_rhs(n_rhs_), dtime(dtime_) {} PTD_STD_DEF(VdShocDecompandSolveData, 5, shcol, nlev, nlevi, dtime, n_rhs); @@ -1069,23 +931,17 @@ struct ComputeShocTempData : public PhysicsTestData { PTD_STD_DEF(ComputeShocTempData, 2, shcol, nlev); }; -// Glue functions to call fortran from from C++ with the Data struct +// Glue functions to call from host with the Data struct void shoc_grid (ShocGridData& d); void shoc_diag_obklen (ShocDiagObklenData& d); void update_host_dse (UpdateHostDseData& d); void shoc_energy_fixer (ShocEnergyFixerData& d); void shoc_energy_integrals (ShocEnergyIntegralsData& d); -void shoc_energy_total_fixer (ShocEnergyTotalFixerData& d); -void shoc_energy_threshold_fixer (ShocEnergyThresholdFixerData& d); -void shoc_energy_dse_fixer (ShocEnergyDseFixerData& d); void calc_shoc_vertflux (CalcShocVertfluxData& d); void calc_shoc_varorcovar (CalcShocVarorcovarData& d); void compute_tmpi (ComputeTmpiData& d); void dp_inverse (DpInverseData& d); -void sfc_fluxes (SfcFluxesData& d); -void impli_srf_stress_term (ImpliSrfStressTermData& d); -void tke_srf_flux_term (TkeSrfFluxTermData& d); void integ_column_stability (IntegColumnStabilityData& d); void check_tke (CheckTkeData& d); void shoc_tke (ShocTkeData& d); @@ -1098,12 +954,6 @@ void compute_brunt_shoc_length (ComputeBruntShocLengthData& void compute_l_inf_shoc_length (ComputeLInfShocLengthData& d); void compute_shoc_mix_shoc_length (ComputeShocMixShocLengthData& d); void check_length_scale_shoc_length (CheckLengthScaleShocLengthData& d); -void fterms_input_for_diag_third_shoc_moment (FtermsInputForDiagThirdShocMomentData& d); -void aa_terms_diag_third_shoc_moment (AaTermsDiagThirdShocMomentData& d); -void f0_to_f5_diag_third_shoc_moment (F0ToF5DiagThirdShocMomentData& d); -void omega_terms_diag_third_shoc_moment (OmegaTermsDiagThirdShocMomentData& d); -void x_y_terms_diag_third_shoc_moment (XYTermsDiagThirdShocMomentData& d); -void w3_diag_third_shoc_moment (W3DiagThirdShocMomentData& d); void clipping_diag_third_shoc_moments (ClippingDiagThirdShocMomentsData& d); void diag_second_moments_srf (DiagSecondMomentsSrfData& d); void linear_interp (LinearInterpData& d); @@ -1131,94 +981,94 @@ void diag_second_shoc_moments (DiagSecondShocMomentsData& void compute_shoc_vapor (ComputeShocVaporData& d); void update_prognostics_implicit (UpdatePrognosticsImplicitData& d); void shoc_main (ShocMainData& d); -void shoc_main_with_init (ShocMainData& d); void pblintd_height (PblintdHeightData& d); void vd_shoc_decomp_and_solve (VdShocDecompandSolveData& d); void pblintd_surf_temp(PblintdSurfTempData& d); void pblintd_check_pblh(PblintdCheckPblhData& d); void pblintd(PblintdData& d); void compute_shoc_temperature(ComputeShocTempData& d); -extern "C" { // _f function decls -void calc_shoc_varorcovar_f(Int shcol, Int nlev, Int nlevi, Real tunefac, +// Call from host + +void calc_shoc_varorcovar_host(Int shcol, Int nlev, Int nlevi, Real tunefac, Real *isotropy_zi, Real *tkh_zi, Real *dz_zi, Real *invar1, Real *invar2, Real *varorcovar); -void calc_shoc_vertflux_f(Int shcol, Int nlev, Int nlevi, Real *tkh_zi, +void calc_shoc_vertflux_host(Int shcol, Int nlev, Int nlevi, Real *tkh_zi, Real *dz_zi, Real *invar, Real *vertflux); -void shoc_diag_second_moments_srf_f(Int shcol, Real* wthl, Real* uw, Real* vw, +void shoc_diag_second_moments_srf_host(Int shcol, Real* wthl, Real* uw, Real* vw, Real* ustar2, Real* wstar); -void shoc_diag_second_moments_ubycond_f(Int shcol, Real* thl, Real* qw, Real* wthl, +void shoc_diag_second_moments_ubycond_host(Int shcol, Real* thl, Real* qw, Real* wthl, Real* wqw, Real* qwthl, Real* uw, Real* vw, Real* wtke); -void update_host_dse_f(Int shcol, Int nlev, Real* thlm, Real* shoc_ql, Real* inv_exner, Real* zt_grid, +void update_host_dse_host(Int shcol, Int nlev, Real* thlm, Real* shoc_ql, Real* inv_exner, Real* zt_grid, Real* phis, Real* host_dse); -void compute_diag_third_shoc_moment_f(Int shcol, Int nlev, Int nlevi, Real* w_sec, +void compute_diag_third_shoc_moment_host(Int shcol, Int nlev, Int nlevi, Real* w_sec, Real* thl_sec, Real* wthl_sec, Real* tke, Real* dz_zt, Real* dz_zi, Real* isotropy_zi, Real* brunt_zi, Real* w_sec_zi, Real* thetal_zi, Real* w3); -void shoc_pblintd_init_pot_f(Int shcol, Int nlev, Real* thl, Real* ql, Real* q, Real* thv); -void compute_shoc_mix_shoc_length_f(Int nlev, Int shcol, Real* tke, Real* brunt, +void shoc_pblintd_init_pot_host(Int shcol, Int nlev, Real* thl, Real* ql, Real* q, Real* thv); +void compute_shoc_mix_shoc_length_host(Int nlev, Int shcol, Real* tke, Real* brunt, Real* zt_grid, Real* l_inf, Real* shoc_mix); -void check_tke_f(Int shcol, Int nlev, Real* tke); -void linear_interp_f(Real* x1, Real* x2, Real* y1, Real* y2, Int km1, Int km2, Int ncol, Real minthresh); -void clipping_diag_third_shoc_moments_f(Int nlevi, Int shcol, Real *w_sec_zi, +void check_tke_host(Int shcol, Int nlev, Real* tke); +void linear_interp_host(Real* x1, Real* x2, Real* y1, Real* y2, Int km1, Int km2, Int ncol, Real minthresh); +void clipping_diag_third_shoc_moments_host(Int nlevi, Int shcol, Real *w_sec_zi, Real *w3); -void shoc_energy_integrals_f(Int shcol, Int nlev, Real *host_dse, Real *pdel, +void shoc_energy_integrals_host(Int shcol, Int nlev, Real *host_dse, Real *pdel, Real *rtm, Real *rcm, Real *u_wind, Real *v_wind, Real *se_int, Real *ke_int, Real *wv_int, Real *wl_int); -void compute_brunt_shoc_length_f(Int nlev, Int nlevi, Int shcol, Real* dz_zt, Real* thv, +void compute_brunt_shoc_length_host(Int nlev, Int nlevi, Int shcol, Real* dz_zt, Real* thv, Real* thv_zi, Real* brunt); -void compute_l_inf_shoc_length_f(Int nlev, Int shcol, Real *zt_grid, Real *dz_zt, +void compute_l_inf_shoc_length_host(Int nlev, Int shcol, Real *zt_grid, Real *dz_zt, Real *tke, Real *l_inf); -void check_length_scale_shoc_length_f(Int nlev, Int shcol, Real* host_dx, Real* host_dy, +void check_length_scale_shoc_length_host(Int nlev, Int shcol, Real* host_dx, Real* host_dy, Real* shoc_mix); -void diag_second_moments_lbycond_f(Int shcol, Real* wthl_sfc, Real* wqw_sfc, Real* uw_sfc, Real* vw_sfc, Real* ustar2, Real* wstar, +void diag_second_moments_lbycond_host(Int shcol, Real* wthl_sfc, Real* wqw_sfc, Real* uw_sfc, Real* vw_sfc, Real* ustar2, Real* wstar, Real* wthl_sec, Real* wqw_sec, Real* uw_sec, Real* vw_sec, Real* wtke_sec, Real* thl_sec, Real* qw_sec, Real* qwthl_sec); -void diag_second_moments_f(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* qw, Real* u_wind, Real* v_wind, Real* tke, Real* isotropy, +void diag_second_moments_host(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* qw, Real* u_wind, Real* v_wind, Real* tke, Real* isotropy, Real* tkh, Real* tk, Real* dz_zi, Real* zt_grid, Real* zi_grid, Real* shoc_mix, Real* thl_sec, Real* qw_sec, Real* wthl_sec, Real* wqw_sec, Real* qwthl_sec, Real* uw_sec, Real* vw_sec, Real* wtke_sec, Real* w_sec); -void diag_second_shoc_moments_f(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* qw, Real* u_wind, Real* v_wind, Real* tke, +void diag_second_shoc_moments_host(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* qw, Real* u_wind, Real* v_wind, Real* tke, Real* isotropy, Real* tkh, Real* tk, Real* dz_zi, Real* zt_grid, Real* zi_grid, Real* shoc_mix, Real* wthl_sfc, Real* wqw_sfc, Real* uw_sfc, Real* vw_sfc, Real* thl_sec, Real* qw_sec, Real* wthl_sec, Real* wqw_sec, Real* qwthl_sec, Real* uw_sec, Real* vw_sec, Real* wtke_sec, Real* w_sec); -void shoc_diag_obklen_f(Int shcol, Real* uw_sfc, Real* vw_sfc, Real* wthl_sfc, Real* wqw_sfc, +void shoc_diag_obklen_host(Int shcol, Real* uw_sfc, Real* vw_sfc, Real* wthl_sfc, Real* wqw_sfc, Real* thl_sfc, Real* cldliq_sfc, Real* qv_sfc, Real* ustar, Real* kbfs, Real* obklen); -void shoc_pblintd_cldcheck_f(Int shcol, Int nlev, Int nlevi, Real* zi, Real* cldn, Real* pblh); -void compute_shr_prod_f(Int nlevi, Int nlev, Int shcol, Real* dz_zi, Real* u_wind, Real* v_wind, Real* sterm); -void shoc_length_f(Int shcol, Int nlev, Int nlevi, Real* host_dx, Real* host_dy, +void shoc_pblintd_cldcheck_host(Int shcol, Int nlev, Int nlevi, Real* zi, Real* cldn, Real* pblh); +void compute_shr_prod_host(Int nlevi, Int nlev, Int shcol, Real* dz_zi, Real* u_wind, Real* v_wind, Real* sterm); +void shoc_length_host(Int shcol, Int nlev, Int nlevi, Real* host_dx, Real* host_dy, Real* zt_grid, Real* zi_grid, Real*dz_zt, Real* tke, Real* thv, Real*brunt, Real* shoc_mix); -void shoc_energy_fixer_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Real* zt_grid, +void shoc_energy_fixer_host(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Real* zt_grid, Real* zi_grid, Real* se_b, Real* ke_b, Real* wv_b, Real* wl_b, Real* se_a, Real* ke_a, Real* wv_a, Real* wl_a, Real* wthl_sfc, Real* wqw_sfc, Real* rho_zt, Real* tke, Real* pint, Real* host_dse); -void compute_shoc_vapor_f(Int shcol, Int nlev, Real* qw, Real* ql, Real* qv); -void update_prognostics_implicit_f(Int shcol, Int nlev, Int nlevi, Int num_tracer, Real dtime, +void compute_shoc_vapor_host(Int shcol, Int nlev, Real* qw, Real* ql, Real* qv); +void update_prognostics_implicit_host(Int shcol, Int nlev, Int nlevi, Int num_tracer, Real dtime, Real* dz_zt, Real* dz_zi, Real* rho_zt, Real* zt_grid, Real* zi_grid, Real* tk, Real* tkh, Real* uw_sfc, Real* vw_sfc, Real* wthl_sfc, Real* wqw_sfc, Real* wtracer_sfc, Real* thetal, Real* qw, Real* tracer, Real* tke, Real* u_wind, Real* v_wind); -void diag_third_shoc_moments_f(Int shcol, Int nlev, Int nlevi, Real* w_sec, Real* thl_sec, +void diag_third_shoc_moments_host(Int shcol, Int nlev, Int nlevi, Real* w_sec, Real* thl_sec, Real* wthl_sec, Real* isotropy, Real* brunt, Real* thetal, Real* tke, Real* dz_zt, Real* dz_zi, Real* zt_grid, Real* zi_grid, Real* w3); -void adv_sgs_tke_f(Int nlev, Int shcol, Real dtime, Real* shoc_mix, Real* wthv_sec, Real* sterm_zt, +void adv_sgs_tke_host(Int nlev, Int shcol, Real dtime, Real* shoc_mix, Real* wthv_sec, Real* sterm_zt, Real* tk, Real* tke, Real* a_diss); -void shoc_assumed_pdf_f(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* qw, Real* w_field, +void shoc_assumed_pdf_host(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* qw, Real* w_field, Real* thl_sec, Real* qw_sec, Real* wthl_sec, Real* w_sec, Real* wqw_sec, Real* qwthl_sec, Real* w3, Real* pres, Real* zt_grid, Real* zi_grid, Real* shoc_cldfrac, Real* shoc_ql, Real* wqls, Real* wthv_sec, Real* shoc_ql2); -void compute_tmpi_f(Int nlevi, Int shcol, Real dtime, Real *rho_zi, Real *dz_zi, Real *tmpi); -void integ_column_stability_f(Int nlev, Int shcol, Real *dz_zt, +void compute_tmpi_host(Int nlevi, Int shcol, Real dtime, Real *rho_zi, Real *dz_zi, Real *tmpi); +void integ_column_stability_host(Int nlev, Int shcol, Real *dz_zt, Real *pres, Real *brunt, Real *brunt_int); -void isotropic_ts_f(Int nlev, Int shcol, Real* brunt_int, Real* tke, +void isotropic_ts_host(Int nlev, Int shcol, Real* brunt_int, Real* tke, Real* a_diss, Real* brunt, Real* isotropy); -void dp_inverse_f(Int nlev, Int shcol, Real *rho_zt, Real *dz_zt, Real *rdp_zt); +void dp_inverse_host(Int nlev, Int shcol, Real *rho_zt, Real *dz_zt, Real *rdp_zt); -int shoc_init_f(Int nlev, Real* pref_mid, Int nbot_shoc, Int ntop_shoc); -Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, Real* host_dx, Real* host_dy, Real* thv, +int shoc_init_host(Int nlev, Real* pref_mid, Int nbot_shoc, Int ntop_shoc); +Int shoc_main_host(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, Real* host_dx, Real* host_dy, Real* thv, Real* zt_grid, Real* zi_grid, Real* pres, Real* presi, Real* pdel, Real* wthl_sfc, Real* wqw_sfc, Real* uw_sfc, Real* vw_sfc, Real* wtracer_sfc, Int num_qtracers, Real* w_field, Real* inv_exner, Real* phis, Real* host_dse, Real* tke, Real* thetal, Real* qw, Real* u_wind, Real* v_wind, @@ -1227,24 +1077,51 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, Real* wthl_sec, Real* wqw_sec, Real* wtke_sec, Real* uw_sec, Real* vw_sec, Real* w3, Real* wqls_sec, Real* brunt, Real* shoc_ql2); -void pblintd_height_f(Int shcol, Int nlev, Int npbl, Real* z, Real* u, Real* v, Real* ustar, Real* thv, Real* thv_ref, Real* pblh, Real* rino, bool* check); +void pblintd_height_host(Int shcol, Int nlev, Int npbl, Real* z, Real* u, Real* v, Real* ustar, Real* thv, Real* thv_ref, Real* pblh, Real* rino, bool* check); -void vd_shoc_decomp_and_solve_f(Int shcol, Int nlev, Int nlevi, Int num_rhs, Real* kv_term, Real* tmpi, Real* rdp_zt, Real dtime, - Real* flux, Real* var); +void vd_shoc_decomp_and_solve_host(Int shcol, Int nlev, Int nlevi, Int num_rhs, Real dtime, Real* kv_term, Real* tmpi, Real* rdp_zt, Real* flux, Real* var); -void pblintd_surf_temp_f(Int shcol, Int nlev, Int nlevi, Real* z, Real* ustar, Real* obklen, Real* kbfs, Real* thv, Real* tlv, Real* pblh, bool* check, Real* rino); -void pblintd_check_pblh_f(Int shcol, Int nlev, Int nlevi, Int npbl, Real* z, Real* ustar, bool* check, Real* pblh); +void pblintd_surf_temp_host(Int shcol, Int nlev, Int nlevi, Real* z, Real* ustar, Real* obklen, Real* kbfs, Real* thv, Real* tlv, Real* pblh, bool* check, Real* rino); -void pblintd_f(Int shcol, Int nlev, Int nlevi, Int npbl, Real* z, Real* zi, Real* thl, Real* ql, Real* q, Real* u, Real* v, Real* ustar, Real* obklen, Real* kbfs, Real* cldn, Real* pblh); -void shoc_grid_f(Int shcol, Int nlev, Int nlevi, Real* zt_grid, Real* zi_grid, Real* pdel, Real* dz_zt, Real* dz_zi, Real* rho_zt); -void eddy_diffusivities_f(Int nlev, Int shcol, Real* pblh, Real* zt_grid, Real* tabs, Real* shoc_mix, Real* sterm_zt, Real* isotropy, +void pblintd_check_pblh_host(Int shcol, Int nlev, Int nlevi, Int npbl, Real* z, Real* ustar, bool* check, Real* pblh); + +void pblintd_host(Int shcol, Int nlev, Int nlevi, Int npbl, Real* z, Real* zi, Real* thl, Real* ql, Real* q, Real* u, Real* v, Real* ustar, Real* obklen, Real* kbfs, Real* cldn, Real* pblh); +void shoc_grid_host(Int shcol, Int nlev, Int nlevi, Real* zt_grid, Real* zi_grid, Real* pdel, Real* dz_zt, Real* dz_zi, Real* rho_zt); +void eddy_diffusivities_host(Int nlev, Int shcol, Real* pblh, Real* zt_grid, Real* tabs, Real* shoc_mix, Real* sterm_zt, Real* isotropy, Real* tke, Real* tkh, Real* tk); -void shoc_tke_f(Int shcol, Int nlev, Int nlevi, Real dtime, Real* wthv_sec, Real* shoc_mix, Real* dz_zi, Real* dz_zt, Real* pres, +void shoc_tke_host(Int shcol, Int nlev, Int nlevi, Real dtime, Real* wthv_sec, Real* shoc_mix, Real* dz_zi, Real* dz_zt, Real* pres, Real* u_wind, Real* v_wind, Real* brunt, Real* obklen, Real* zt_grid, Real* zi_grid, Real* pblh, Real* tke, Real* tk, Real* tkh, Real* isotropy); -void compute_shoc_temperature_f(Int shcol, Int nlev, Real* thetal, Real* ql, Real* inv_exner, Real* tabs); -} // end _f function decls +void compute_shoc_temperature_host(Int shcol, Int nlev, Real* thetal, Real* ql, Real* inv_exner, Real* tabs); + +void shoc_energy_total_fixer_host(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Real* zt_grid, Real* zi_grid, Real* se_b, Real* ke_b, Real* wv_b, Real* wl_b, Real* se_a, Real* ke_a, Real* wv_a, Real* wl_a, Real* wthl_sfc, Real* wqw_sfc, Real* rho_zt, Real* pint, Real* te_a, Real* te_b); + +void shoc_assumed_pdf_tilde_to_real_host(Real w_first, Real sqrtw2, Real* w1); + +void shoc_assumed_pdf_vv_parameters_host(Real w_first, Real w_sec, Real w3var, Real w_tol_sqd, Real* skew_w, Real* w1_1, Real* w1_2, Real* w2_1, Real* w2_2, Real* a); + +void shoc_assumed_pdf_thl_parameters_host(Real wthlsec, Real sqrtw2, Real sqrtthl, Real thlsec, Real thl_first, Real w1_1, Real w1_2, Real skew_w, Real a, Real thl_tol, Real w_thresh, Real* thl1_1, Real* thl1_2, Real* thl2_1, Real* thl2_2, Real* sqrtthl2_1, Real* sqrtthl2_2); + +void shoc_assumed_pdf_qw_parameters_host(Real wqwsec, Real sqrtw2, Real skew_w, Real sqrtqt, Real qwsec, Real w1_2, Real w1_1, Real qw_first, Real a, Real rt_tol, Real w_thresh, Real* qw1_1, Real* qw1_2, Real* qw2_1, Real* qw2_2, Real* sqrtqw2_1, Real* sqrtqw2_2); + +void shoc_assumed_pdf_inplume_correlations_host(Real sqrtqw2_1, Real sqrtthl2_1, Real a, Real sqrtqw2_2, Real sqrtthl2_2, Real qwthlsec, Real qw1_1, Real qw_first, Real thl1_1, Real thl_first, Real qw1_2, Real thl1_2, Real* r_qwthl_1); + +void shoc_assumed_pdf_compute_temperature_host(Real thl1, Real pval, Real* tl1); + +void shoc_assumed_pdf_compute_qs_host(Real tl1_1, Real tl1_2, Real pval, Real* qs1, Real* beta1, Real* qs2, Real* beta2); + +void shoc_assumed_pdf_compute_s_host(Real qw1, Real qs1, Real beta, Real pval, Real thl2, Real qw2, Real sqrtthl2, Real sqrtqw2, Real r_qwthl, Real* s, Real* std_s, Real* qn, Real* c); + +void shoc_assumed_pdf_compute_sgs_liquid_host(Real a, Real ql1, Real ql2, Real* shoc_ql); + +void shoc_assumed_pdf_compute_cloud_liquid_variance_host(Real a, Real s1, Real ql1, Real c1, Real std_s1, Real s2, Real ql2, Real c2, Real std_s2, Real shoc_ql, Real* shoc_ql2); + +void shoc_assumed_pdf_compute_liquid_water_flux_host(Real a, Real w1_1, Real w_first, Real ql1, Real w1_2, Real ql2, Real* wqls); + +void shoc_assumed_pdf_compute_buoyancy_flux_host(Real wthlsec, Real wqwsec, Real pval, Real wqls, Real* wthv_sec); + +// end _host function decls } // namespace shoc } // namespace scream diff --git a/components/eamxx/src/physics/shoc/tests/shoc_unit_tests_common.hpp b/components/eamxx/src/physics/shoc/tests/infra/shoc_unit_tests_common.hpp similarity index 66% rename from components/eamxx/src/physics/shoc/tests/shoc_unit_tests_common.hpp rename to components/eamxx/src/physics/shoc/tests/infra/shoc_unit_tests_common.hpp index 0d74ebaa5fe1..f7326c06df95 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_unit_tests_common.hpp +++ b/components/eamxx/src/physics/shoc/tests/infra/shoc_unit_tests_common.hpp @@ -4,6 +4,9 @@ #include "shoc_functions.hpp" #include "share/scream_types.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" +#include "share/util/scream_setup_random_test.hpp" +#include "ekat/util/ekat_file_utils.hpp" +#include "ekat/util/ekat_test_utils.hpp" namespace scream { namespace shoc { @@ -21,6 +24,12 @@ namespace unit_test { struct UnitWrap { + enum BASELINE_ACTION { + NONE, + COMPARE, + GENERATE + }; + template struct UnitTest : public KokkosTypes { @@ -48,8 +57,68 @@ struct UnitWrap { using Smask = typename Functions::Smask; using C = typename Functions::C; - static constexpr Int max_pack_size = 16; - static constexpr Int num_test_itrs = max_pack_size / Spack::n; + struct Base { + std::string m_baseline_path; + std::string m_test_name; + BASELINE_ACTION m_baseline_action; + ekat::FILEPtr m_fid; + + Base() : + m_baseline_path(""), + m_test_name(Catch::getResultCapture().getCurrentTestName()), + m_baseline_action(NONE), + m_fid() + { + //Functions::shoc_init(); // just in case there is ever global shoc data + auto& ts = ekat::TestSession::get(); + if (ts.flags["c"]) { + m_baseline_action = COMPARE; + } + else if (ts.flags["g"]) { + m_baseline_action = GENERATE; + } + else if (ts.flags["n"]) { + m_baseline_action = NONE; + } + m_baseline_path = ts.params["b"]; + + + EKAT_REQUIRE_MSG( !(m_baseline_action != NONE && m_baseline_path == ""), + "SHOC unit test flags problem: baseline actions were requested but no baseline path was provided"); + + std::string baseline_name = m_baseline_path + "/" + m_test_name; + if (m_baseline_action == COMPARE) { + m_fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + } + else if (m_baseline_action == GENERATE) { + m_fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + } + } + + ~Base() + { + } + + std::mt19937_64 get_engine() + { + if (m_baseline_action != COMPARE) { + // We can use any seed + int seed; + auto engine = setup_random_test(nullptr, &seed); + if (m_baseline_action == GENERATE) { + // Write the seed + ekat::write(&seed, 1, m_fid); + } + return engine; + } + else { + // Read the seed + int seed; + ekat::read(&seed, 1, m_fid); + return setup_random_test(seed); + } + } + }; // Put struct decls here struct TestCalcShocVertflux; diff --git a/components/eamxx/src/physics/shoc/tests/shoc_aa_diag_third_moms_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_aa_diag_third_moms_tests.cpp deleted file mode 100644 index d6d7d522d66a..000000000000 --- a/components/eamxx/src/physics/shoc/tests/shoc_aa_diag_third_moms_tests.cpp +++ /dev/null @@ -1,110 +0,0 @@ -#include "catch2/catch.hpp" - -#include "shoc_unit_tests_common.hpp" -#include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" -#include "physics/share/physics_constants.hpp" -#include "share/scream_types.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/util/ekat_arch.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" - -#include -#include -#include -#include - -namespace scream { -namespace shoc { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestAAdiagThirdMoms { - - static void run_property() - { - - // Tests for the SHOC function: - // aa_terms_diag_third_shoc_moment - - // Run test two times. One where 0 parameters stay the same - // and the other where 1 parameters vary. Verify that the aa0 - // term remains constant between the two tests. - - // omega0 term - constexpr static Real omega0 = 7.54e-2; - // omega1 term - constexpr static Real omega1 = 5.37e-3; - // omega2 term - constexpr static Real omega2 = 0.54; - // x0 term - constexpr static Real x0 = -4.31; - // y0 term - constexpr static Real y0 = 22.72; - // x1 term - constexpr static Real x1_test1a = 41.05; - // y1 term - constexpr static Real y1_test1a = 375.69; - - // Initialize data structure for bridging to F90 - AaTermsDiagThirdShocMomentData SDS; - - // Load up the data - SDS.omega0 = omega0; - SDS.x0 = x0; - SDS.y0 = y0; - SDS.omega1 = omega1; - SDS.x1 = x1_test1a; - SDS.y1 = y1_test1a; - SDS.omega2 = omega2; - - // Call the fortran implementation - aa_terms_diag_third_shoc_moment(SDS); - - // Save the output - Real aa0_test1a = SDS.aa0; - Real aa1_test1a = SDS.aa1; - - // Load up data where only the 1 terms have varies - SDS.x1 = 0.3*x1_test1a; - SDS.y1 = 0.3*y1_test1a; - - // Call the fortran implementation - aa_terms_diag_third_shoc_moment(SDS); - - // Check the result - REQUIRE(SDS.aa0 == aa0_test1a); - REQUIRE(SDS.aa1 < aa1_test1a); - - } - - static void run_bfb() - { - // TODO - } - -}; - -} // namespace unit_test -} // namespace shoc -} // namespace scream - -namespace{ - -TEST_CASE("shoc_aa_diag_third_moms_property", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestAAdiagThirdMoms; - - TestStruct::run_property(); -} - -TEST_CASE("shoc_aa_diag_third_moms_bfb", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestAAdiagThirdMoms; - - TestStruct::run_bfb(); - -} - -} // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp index e947946fa203..93926020d82f 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp @@ -1,504 +1,481 @@ -#include "catch2/catch.hpp" - -#include "shoc_unit_tests_common.hpp" -#include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" -#include "physics/share/physics_constants.hpp" -#include "share/scream_types.hpp" -#include "share/util/scream_setup_random_test.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/util/ekat_arch.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" - -#include -#include -#include -#include - -namespace scream { -namespace shoc { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestShocAssumedPdf { - - static void run_property() - { - static constexpr Int shcol = 2; - static constexpr Int nlev = 5; - static constexpr auto nlevi = nlev + 1; - - // Tests for the top level subroutine - // shoc_assumed_pdf - - // Tests will start simple, and gradually add complexity to test - // the physics. - // NOTE: for this test we want exactly two columns. - - // TEST ONE - // No SGS variability test. Given inputs where there is a saturated - // profile but NO SGS variability in the scalar fluxes or variances - // (i.e. all second and third moment terms are zero), then verify that - // that cloud fraction is either 1 or 0 and that the SGS variability - // outputs are also zero everywhere. - - // Define input data - - // Note that the moisture and height profiles below represent that - // of the BOMEX case, but the temperatures are much colder, to encourage - // there to be points with ample cloud produced for this test. - - // Liquid water potential temperature [K] - static constexpr Real thetal[nlev] = {303, 300, 298, 298, 300}; - // Total water mixing ratio [kg/kg] - static constexpr Real qw[nlev] = {0.003, 0.004, 0.011, 0.016, 0.017}; - // Pressure [Pa] - static constexpr Real pres[nlev] = {70000, 80000, 85000, 90000, 100000}; - // Define the heights on the zt grid [m] - static constexpr Real zi_grid[nlevi] = {3000, 2000, 1500, 1000, 500, 0}; - - // All variances will be given zero or minimum threshold inputs - - // Define some reasonable bounds for output - static constexpr Real wqls_bound = 0.1; - static constexpr Real wthv_sec_bound = 10; - static constexpr Real shoc_ql2_bound = 0.1; - static constexpr Real shoc_ql_bound = 0.1; - - Real zt_grid[nlev]; - // Compute heights on midpoint grid - for(Int n = 0; n < nlev; ++n) { - zt_grid[n] = 0.5*(zi_grid[n]+zi_grid[n+1]); - } - - // Initialize data structure for bridging to F90 - ShocAssumedPdfData SDS(shcol, nlev, nlevi); - - // Load input data - for(Int s = 0; s < shcol; ++s) { - for(Int n = 0; n < nlev; ++n) { - const auto offset = n + s * nlev; - - SDS.thetal[offset] = thetal[n]; - SDS.qw[offset] = qw[n]; - SDS.pres[offset] = pres[n]; - SDS.zt_grid[offset] = zt_grid[n]; - SDS.w_field[offset] = 0; - SDS.w_sec[offset] = 0.004; - } - - for(Int n = 0; n < nlevi; ++n) { - const auto offset = n + s * nlevi; - - SDS.thl_sec[offset] = 0; - SDS.qw_sec[offset] = 0; - SDS.wthl_sec[offset] = 0; - SDS.wqw_sec[offset] = 0; - SDS.qwthl_sec[offset] = 0; - SDS.w3[offset] = 0; - SDS.zi_grid[offset] = zi_grid[n]; - } - } - - // Check that the inputs make sense - for(Int s = 0; s < shcol; ++s) { - for(Int n = 0; n < nlev; ++n) { - const auto offset = n + s * nlev; - - REQUIRE(SDS.qw[offset] > 0); - REQUIRE(SDS.thetal[offset] > 0); - REQUIRE(SDS.pres[offset] > 0); - REQUIRE(SDS.zt_grid[offset] > 0); - } - - for(Int n = 0; n < nlevi; ++n) { - const auto offset = n + s * nlev; - - REQUIRE(SDS.zi_grid[offset] >= 0); - } - } - - // Check that zt increase updward and pres decrease upward - for(Int s = 0; s < shcol; ++s) { - for(Int n = 0; n < nlev - 1; ++n) { - const auto offset = n + s * nlev; - REQUIRE(SDS.zt_grid[offset + 1] - SDS.zt_grid[offset] < 0); - REQUIRE(SDS.pres[offset + 1] - SDS.pres[offset] > 0); - } - - // Check that zi increase upward - for(Int n = 0; n < nlevi - 1; ++n) { - const auto offset = n + s * nlevi; - REQUIRE(SDS.zi_grid[offset + 1] - SDS.zi_grid[offset] < 0); - } - - } - - // Test that the inputs are reasonable. - REQUIRE(SDS.nlevi - SDS.nlev == 1); - // For this test we want exactly two columns - REQUIRE(SDS.shcol == 2); - - // Call the C++ implementation. - SDS.transpose(); - // expects data in fortran layout - shoc_assumed_pdf_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.thetal, SDS.qw, SDS.w_field, - SDS.thl_sec, SDS.qw_sec, SDS.wthl_sec, SDS.w_sec, SDS.wqw_sec, - SDS.qwthl_sec, SDS.w3, SDS.pres, SDS.zt_grid, SDS.zi_grid, - SDS.shoc_cldfrac, SDS.shoc_ql, SDS.wqls, SDS.wthv_sec, SDS.shoc_ql2); - SDS.transpose(); - - // Verify the result - // Make sure cloud fraction is either 1 or 0 and all - // SGS terms are zero. - - for(Int s = 0; s < shcol; ++s) { - for(Int n = 0; n < nlev; ++n) { - const auto offset = n + s * nlev; - - REQUIRE( (SDS.shoc_cldfrac[offset] == 0 || SDS.shoc_cldfrac[offset] == 1) ); - REQUIRE(SDS.wqls[offset] == 0); - REQUIRE(SDS.wthv_sec[offset] == 0); - REQUIRE(std::abs(SDS.shoc_ql2[offset]) < std::numeric_limits::epsilon()); // Computation is not exactly BFB with 0 - REQUIRE(SDS.shoc_ql[offset] >= 0); - } - } - - // TEST TWO - // Add in Scalar fluxes. This should give us a nonzero - // buoyancy flux everywhere but other SGS terms should remain zero - - // We will assume turbulence with a uniform profile - - // Flux of liquid water [K m/s] - static constexpr Real wthl_sec = -0.03; - // Flux of total water [m/s kg/kg] - static constexpr Real wqw_sec = 0.0002; - - for(Int s = 0; s < shcol; ++s) { - for(Int n = 0; n < nlevi; ++n) { - const auto offset = n + s * nlevi; - - SDS.wthl_sec[offset] = wthl_sec; - SDS.wqw_sec[offset] = wqw_sec; - } - } - - // Call the C++ implementation. - SDS.transpose(); - // expects data in fortran layout - shoc_assumed_pdf_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.thetal, SDS.qw, SDS.w_field, - SDS.thl_sec, SDS.qw_sec, SDS.wthl_sec, SDS.w_sec, SDS.wqw_sec, - SDS.qwthl_sec, SDS.w3, SDS.pres, SDS.zt_grid, SDS.zi_grid, - SDS.shoc_cldfrac, SDS.shoc_ql, SDS.wqls, SDS.wthv_sec, SDS.shoc_ql2); - SDS.transpose(); - - // Verify the result - // Make sure cloud fraction is either 1 or 0 and all - // SGS terms are zero, EXCEPT wthv_sec. - - for(Int s = 0; s < shcol; ++s) { - for(Int n = 0; n < nlev; ++n) { - const auto offset = n + s * nlev; - - REQUIRE( (SDS.shoc_cldfrac[offset] == 0 || SDS.shoc_cldfrac[offset] == 1) ); - REQUIRE(SDS.wqls[offset] == 0); - REQUIRE(SDS.wthv_sec[offset] != 0); - REQUIRE(std::abs(SDS.wthv_sec[offset] < wthv_sec_bound)); - REQUIRE(std::abs(SDS.shoc_ql2[offset]) < std::numeric_limits::epsilon()); // Computation is not exactly BFB with 0 - REQUIRE(SDS.shoc_ql[offset] >= 0); - REQUIRE(SDS.shoc_ql[offset] < shoc_ql_bound); - } - } - - // TEST THREE and FOUR - // Add in Scalar variances, and POSITIVE vertical velocity skewness test. - - // Add strong scalar variances as such that will produce cloud at every level. - - // For the first column feed in zero vertical velocity skewness. - // For the second column feed in large veriticle velocity skewss. - // Verify that for points where cloud fraction was < 0.5 in the first column, - // that cloud fraction then vice versa for points with cloud fraction > 0.5. - - // Thetal variance [K^2] - static constexpr Real thl_sec = 2; - // total water variance [kg^2/kg^2] - static constexpr Real qw_sec = 0.0002; - // Temperature and total water covariance [K kg/kg] - static constexpr Real qwthl_sec = 1e-5; - // Vertical velocity variance [m2/s2] - static constexpr Real w_sec = 0.2; - - // Load input data - for(Int s = 0; s < shcol; ++s) { - for(Int n = 0; n < nlev; ++n) { - const auto offset = n + s * nlev; - - SDS.w_sec[offset] = w_sec; - } - - for(Int n = 0; n < nlevi; ++n) { - const auto offset = n + s * nlevi; - - SDS.thl_sec[offset] = thl_sec; - SDS.qw_sec[offset] = qw_sec; - SDS.qwthl_sec[offset] = qwthl_sec; - SDS.w3[offset] = s*1.0; - } - } - - // Call the C++ implementation. - SDS.transpose(); - // expects data in fortran layout - shoc_assumed_pdf_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.thetal, SDS.qw, SDS.w_field, - SDS.thl_sec, SDS.qw_sec, SDS.wthl_sec, SDS.w_sec, SDS.wqw_sec, - SDS.qwthl_sec, SDS.w3, SDS.pres, SDS.zt_grid, SDS.zi_grid, - SDS.shoc_cldfrac, SDS.shoc_ql, SDS.wqls, SDS.wthv_sec, SDS.shoc_ql2); - SDS.transpose(); - - // Check the result - - // With such a turbulence and scalar profile, this should have - // encouraged cloud everywhere. Verify that this is true. - - // Also verify that output lies within some reasonable bounds. - - // Then verify vertical velocity skewness info - - for(Int s = 0; s < shcol; ++s) { - for(Int n = 0; n < nlevi; ++n) { - const auto offset = n + s * nlevi; - const auto offsets = n + (s+1) * nlevi; - if (s < shcol-1){ - // Verify input w3 is greater in subsequent columns - REQUIRE(SDS.w3[offsets] > SDS.w3[offset]); - } - } - - // Verify output falls within reasonable bounds. For this positive - // vertical velocity skewness test and with the give inputs there - // should be cloud everywhere and all flux terms should be positive. - - for(Int n = 0; n < nlev; ++n) { - const auto offset = n + s * nlev; - const auto offsets = n + (s+1) * nlev; - - REQUIRE( (SDS.shoc_cldfrac[offset] > 0 || SDS.shoc_cldfrac[offset] < 1) ); - REQUIRE(SDS.wqls[offset] > 0); - REQUIRE(SDS.wthv_sec[offset] > 0); - REQUIRE(SDS.shoc_ql2[offset] > 0); - REQUIRE(SDS.shoc_ql[offset] > 0); - - REQUIRE(SDS.wqls[offset] < 0.1); - REQUIRE(SDS.wthv_sec[offset] < wthv_sec_bound); - REQUIRE(SDS.shoc_ql2[offset] < shoc_ql2_bound); - REQUIRE(SDS.shoc_ql[offset] < shoc_ql_bound); - - // Now verify that the relationships in a strongly positive vertical - // velocity flux regime hold true, relative to a symmetric vertical - // velocity regime. - if (s < shcol-1){ - - if (SDS.shoc_cldfrac[offset] < 0.5){ - REQUIRE(SDS.shoc_cldfrac[offsets] < SDS.shoc_cldfrac[offset]); - } - else if (SDS.shoc_cldfrac[offset] > 0.5){ - REQUIRE(SDS.shoc_cldfrac[offsets] > SDS.shoc_cldfrac[offset]); - } - - // In addition, in a positive skewness environment, the following - // should also be true - - // Grid mean liquid water decreased - REQUIRE(SDS.shoc_ql[offsets] < SDS.shoc_ql[offset]); - // liquid water flux increased - REQUIRE(SDS.wqls[offsets] > SDS.wqls[offset]); - // buoyancy flux increased - REQUIRE(SDS.wthv_sec[offsets] > SDS.wthv_sec[offset]); - // liquid water variance increased - REQUIRE(SDS.shoc_ql2[offsets] > SDS.shoc_ql2[offset]); - } - } - } - - // TEST FIVE - // Negative vertical velocity skewness. - - // Using same input as the above test, feed one column with zero skeweness - // and another test with negative vertical velocity skewness and verify - // result is physical. - - // Load input data - for(Int s = 0; s < shcol; ++s) { - for(Int n = 0; n < nlevi; ++n) { - const auto offset = n + s * nlevi; - - SDS.w3[offset] = s*-1.0; - } - } - - // Call the C++ implementation. - SDS.transpose(); - // expects data in fortran layout - shoc_assumed_pdf_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.thetal, SDS.qw, SDS.w_field, - SDS.thl_sec, SDS.qw_sec, SDS.wthl_sec, SDS.w_sec, SDS.wqw_sec, - SDS.qwthl_sec, SDS.w3, SDS.pres, SDS.zt_grid, SDS.zi_grid, - SDS.shoc_cldfrac, SDS.shoc_ql, SDS.wqls, SDS.wthv_sec, SDS.shoc_ql2); - SDS.transpose(); - - // Check the result - - // Verify that output lies within some reasonable bounds. - - // Then verify vertical velocity skewness info - - for(Int s = 0; s < shcol; ++s) { - for(Int n = 0; n < nlevi; ++n) { - const auto offset = n + s * nlevi; - const auto offsets = n + (s+1) * nlevi; - if (s < shcol-1){ - // Verify input w3 is greater in subsequent columns - REQUIRE(SDS.w3[offsets] < SDS.w3[offset]); - } - } - - // Verify output falls within reasonable bounds - // For this negative vertical velocity test some variables - // will be expected to be less than zero. - - for(Int n = 0; n < nlev; ++n) { - const auto offset = n + s * nlev; - const auto offsets = n + (s+1) * nlev; - - REQUIRE( (SDS.shoc_cldfrac[offset] >= 0 || SDS.shoc_cldfrac[offset] < 1) ); - REQUIRE(SDS.shoc_ql2[offset] > 0); - REQUIRE(SDS.shoc_ql[offset] >= 0); - - REQUIRE(std::abs(SDS.wqls[offset] ) < wqls_bound); - REQUIRE(std::abs(SDS.wthv_sec[offset]) < wthv_sec_bound); - REQUIRE(SDS.shoc_ql2[offset] < shoc_ql2_bound); - REQUIRE(SDS.shoc_ql[offset] < shoc_ql_bound); - - // Now verify that the relationships in a strongly negative vertical - // velocity flux regime hold true, relative to a symmetric vertical - // velocity regime. - if (s < shcol-1){ - - if (SDS.shoc_cldfrac[offset] < 0.5){ - REQUIRE(SDS.shoc_cldfrac[offsets] < SDS.shoc_cldfrac[offset]); - } - else if (SDS.shoc_cldfrac[offset] > 0.5){ - REQUIRE(SDS.shoc_cldfrac[offsets] > SDS.shoc_cldfrac[offset]); - } - - // In addition, in a positive skewness environment, the following - // should also be true - - // Grid mean liquid water decreased - REQUIRE(SDS.shoc_ql[offsets] < SDS.shoc_ql[offset]); - // if cloud present, verify liquid water and buoyancy flux is negative - if (SDS.shoc_ql[offsets] > 0){ - REQUIRE(SDS.wqls[offsets] < SDS.wqls[offset]); - REQUIRE(SDS.wqls[offsets] < 0); - - REQUIRE(SDS.wthv_sec[offsets] < SDS.wthv_sec[offset]); - REQUIRE(SDS.wthv_sec[offsets] < 0); - } - - // liquid water variance increased - REQUIRE(SDS.shoc_ql2[offsets] > SDS.shoc_ql2[offset]); - } - } - } - - } - - static void run_bfb() - { - auto engine = setup_random_test(); - - ShocAssumedPdfData SDS_f90[] = { - // shcol, nlev, nlevi - ShocAssumedPdfData(10, 71, 72), - ShocAssumedPdfData(10, 12, 13), - ShocAssumedPdfData(7, 16, 17), - ShocAssumedPdfData(2, 7, 8), - }; - - // Generate random input data - for (auto& d : SDS_f90) { - d.randomize(engine, { - {d.thetal, {500, 700}}, - {d.zi_grid, {0, 100}}, - }); - } - - // Create copies of data for use by cxx. Needs to happen before fortran calls so that - // inout data is in original state - ShocAssumedPdfData SDS_cxx[] = { - ShocAssumedPdfData(SDS_f90[0]), - ShocAssumedPdfData(SDS_f90[1]), - ShocAssumedPdfData(SDS_f90[2]), - ShocAssumedPdfData(SDS_f90[3]), - }; - - // Assume all data is in C layout - - // Get data from fortran - for (auto& d : SDS_f90) { - // expects data in C layout - shoc_assumed_pdf(d); - } - - // Get data from cxx - for (auto& d : SDS_cxx) { - d.transpose(); - // expects data in fortran layout - shoc_assumed_pdf_f(d.shcol, d.nlev, d.nlevi, d.thetal, d.qw, d.w_field, - d.thl_sec, d.qw_sec, d.wthl_sec, d.w_sec, d.wqw_sec, - d.qwthl_sec, d.w3, d.pres, d.zt_grid, d.zi_grid, - d.shoc_cldfrac, d.shoc_ql, d.wqls, d.wthv_sec, d.shoc_ql2); - d.transpose(); - } - - // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(SDS_f90) / sizeof(ShocAssumedPdfData); - for (Int i = 0; i < num_runs; ++i) { - ShocAssumedPdfData& d_f90 = SDS_f90[i]; - ShocAssumedPdfData& d_cxx = SDS_cxx[i]; - for (Int k = 0; k < d_f90.total(d_f90.wqls); ++k) { - REQUIRE(d_f90.shoc_cldfrac[k] == d_cxx.shoc_cldfrac[k]); - REQUIRE(d_f90.shoc_ql[k] == d_cxx.shoc_ql[k]); - REQUIRE(d_f90.wqls[k] == d_cxx.wqls[k]); - REQUIRE(d_f90.wthv_sec[k] == d_cxx.wthv_sec[k]); - REQUIRE(d_f90.shoc_ql2[k] == d_cxx.shoc_ql2[k]); - } - } - } - } -}; - -} // namespace unit_test -} // namespace shoc -} // namespace scream - -namespace { - -TEST_CASE("shoc_assumed_pdf_property", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocAssumedPdf; - - TestStruct::run_property(); -} - -TEST_CASE("shoc_assumed_pdf_bfb", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocAssumedPdf; - - TestStruct::run_bfb(); -} - -} // namespace +#include "catch2/catch.hpp" + +#include "shoc_unit_tests_common.hpp" +#include "shoc_functions.hpp" +#include "shoc_test_data.hpp" +#include "physics/share/physics_constants.hpp" +#include "share/scream_types.hpp" +#include "share/util/scream_setup_random_test.hpp" + +#include "ekat/ekat_pack.hpp" +#include "ekat/util/ekat_arch.hpp" +#include "ekat/kokkos/ekat_kokkos_utils.hpp" + +#include +#include +#include +#include + +namespace scream { +namespace shoc { +namespace unit_test { + +template +struct UnitWrap::UnitTest::TestShocAssumedPdf : public UnitWrap::UnitTest::Base { + + void run_property() + { + static constexpr Int shcol = 2; + static constexpr Int nlev = 5; + static constexpr auto nlevi = nlev + 1; + + // Tests for the top level subroutine + // shoc_assumed_pdf + + // Tests will start simple, and gradually add complexity to test + // the physics. + // NOTE: for this test we want exactly two columns. + + // TEST ONE + // No SGS variability test. Given inputs where there is a saturated + // profile but NO SGS variability in the scalar fluxes or variances + // (i.e. all second and third moment terms are zero), then verify that + // that cloud fraction is either 1 or 0 and that the SGS variability + // outputs are also zero everywhere. + + // Define input data + + // Note that the moisture and height profiles below represent that + // of the BOMEX case, but the temperatures are much colder, to encourage + // there to be points with ample cloud produced for this test. + + // Liquid water potential temperature [K] + static constexpr Real thetal[nlev] = {303, 300, 298, 298, 300}; + // Total water mixing ratio [kg/kg] + static constexpr Real qw[nlev] = {0.003, 0.004, 0.011, 0.016, 0.017}; + // Pressure [Pa] + static constexpr Real pres[nlev] = {70000, 80000, 85000, 90000, 100000}; + // Define the heights on the zt grid [m] + static constexpr Real zi_grid[nlevi] = {3000, 2000, 1500, 1000, 500, 0}; + + // All variances will be given zero or minimum threshold inputs + + // Define some reasonable bounds for output + static constexpr Real wqls_bound = 0.1; + static constexpr Real wthv_sec_bound = 10; + static constexpr Real shoc_ql2_bound = 0.1; + static constexpr Real shoc_ql_bound = 0.1; + + Real zt_grid[nlev]; + // Compute heights on midpoint grid + for(Int n = 0; n < nlev; ++n) { + zt_grid[n] = 0.5*(zi_grid[n]+zi_grid[n+1]); + } + + // Initialize data structure for bridging to F90 + ShocAssumedPdfData SDS(shcol, nlev, nlevi); + + // Load input data + for(Int s = 0; s < shcol; ++s) { + for(Int n = 0; n < nlev; ++n) { + const auto offset = n + s * nlev; + + SDS.thetal[offset] = thetal[n]; + SDS.qw[offset] = qw[n]; + SDS.pres[offset] = pres[n]; + SDS.zt_grid[offset] = zt_grid[n]; + SDS.w_field[offset] = 0; + SDS.w_sec[offset] = 0.004; + } + + for(Int n = 0; n < nlevi; ++n) { + const auto offset = n + s * nlevi; + + SDS.thl_sec[offset] = 0; + SDS.qw_sec[offset] = 0; + SDS.wthl_sec[offset] = 0; + SDS.wqw_sec[offset] = 0; + SDS.qwthl_sec[offset] = 0; + SDS.w3[offset] = 0; + SDS.zi_grid[offset] = zi_grid[n]; + } + } + + // Check that the inputs make sense + for(Int s = 0; s < shcol; ++s) { + for(Int n = 0; n < nlev; ++n) { + const auto offset = n + s * nlev; + + REQUIRE(SDS.qw[offset] > 0); + REQUIRE(SDS.thetal[offset] > 0); + REQUIRE(SDS.pres[offset] > 0); + REQUIRE(SDS.zt_grid[offset] > 0); + } + + for(Int n = 0; n < nlevi; ++n) { + const auto offset = n + s * nlev; + + REQUIRE(SDS.zi_grid[offset] >= 0); + } + } + + // Check that zt increase updward and pres decrease upward + for(Int s = 0; s < shcol; ++s) { + for(Int n = 0; n < nlev - 1; ++n) { + const auto offset = n + s * nlev; + REQUIRE(SDS.zt_grid[offset + 1] - SDS.zt_grid[offset] < 0); + REQUIRE(SDS.pres[offset + 1] - SDS.pres[offset] > 0); + } + + // Check that zi increase upward + for(Int n = 0; n < nlevi - 1; ++n) { + const auto offset = n + s * nlevi; + REQUIRE(SDS.zi_grid[offset + 1] - SDS.zi_grid[offset] < 0); + } + + } + + // Test that the inputs are reasonable. + REQUIRE(SDS.nlevi - SDS.nlev == 1); + // For this test we want exactly two columns + REQUIRE(SDS.shcol == 2); + + // Call the C++ implementation. + shoc_assumed_pdf(SDS); + + // Verify the result + // Make sure cloud fraction is either 1 or 0 and all + // SGS terms are zero. + + for(Int s = 0; s < shcol; ++s) { + for(Int n = 0; n < nlev; ++n) { + const auto offset = n + s * nlev; + + REQUIRE( (SDS.shoc_cldfrac[offset] == 0 || SDS.shoc_cldfrac[offset] == 1) ); + REQUIRE(SDS.wqls[offset] == 0); + REQUIRE(SDS.wthv_sec[offset] == 0); + REQUIRE(std::abs(SDS.shoc_ql2[offset]) < std::numeric_limits::epsilon()); // Computation is not exactly BFB with 0 + REQUIRE(SDS.shoc_ql[offset] >= 0); + } + } + + // TEST TWO + // Add in Scalar fluxes. This should give us a nonzero + // buoyancy flux everywhere but other SGS terms should remain zero + + // We will assume turbulence with a uniform profile + + // Flux of liquid water [K m/s] + static constexpr Real wthl_sec = -0.03; + // Flux of total water [m/s kg/kg] + static constexpr Real wqw_sec = 0.0002; + + for(Int s = 0; s < shcol; ++s) { + for(Int n = 0; n < nlevi; ++n) { + const auto offset = n + s * nlevi; + + SDS.wthl_sec[offset] = wthl_sec; + SDS.wqw_sec[offset] = wqw_sec; + } + } + + // Call the C++ implementation. + shoc_assumed_pdf(SDS); + + // Verify the result + // Make sure cloud fraction is either 1 or 0 and all + // SGS terms are zero, EXCEPT wthv_sec. + + for(Int s = 0; s < shcol; ++s) { + for(Int n = 0; n < nlev; ++n) { + const auto offset = n + s * nlev; + + REQUIRE( (SDS.shoc_cldfrac[offset] == 0 || SDS.shoc_cldfrac[offset] == 1) ); + REQUIRE(SDS.wqls[offset] == 0); + REQUIRE(SDS.wthv_sec[offset] != 0); + REQUIRE(std::abs(SDS.wthv_sec[offset] < wthv_sec_bound)); + REQUIRE(std::abs(SDS.shoc_ql2[offset]) < std::numeric_limits::epsilon()); // Computation is not exactly BFB with 0 + REQUIRE(SDS.shoc_ql[offset] >= 0); + REQUIRE(SDS.shoc_ql[offset] < shoc_ql_bound); + } + } + + // TEST THREE and FOUR + // Add in Scalar variances, and POSITIVE vertical velocity skewness test. + + // Add strong scalar variances as such that will produce cloud at every level. + + // For the first column feed in zero vertical velocity skewness. + // For the second column feed in large veriticle velocity skewss. + // Verify that for points where cloud fraction was < 0.5 in the first column, + // that cloud fraction then vice versa for points with cloud fraction > 0.5. + + // Thetal variance [K^2] + static constexpr Real thl_sec = 2; + // total water variance [kg^2/kg^2] + static constexpr Real qw_sec = 0.0002; + // Temperature and total water covariance [K kg/kg] + static constexpr Real qwthl_sec = 1e-5; + // Vertical velocity variance [m2/s2] + static constexpr Real w_sec = 0.2; + + // Load input data + for(Int s = 0; s < shcol; ++s) { + for(Int n = 0; n < nlev; ++n) { + const auto offset = n + s * nlev; + + SDS.w_sec[offset] = w_sec; + } + + for(Int n = 0; n < nlevi; ++n) { + const auto offset = n + s * nlevi; + + SDS.thl_sec[offset] = thl_sec; + SDS.qw_sec[offset] = qw_sec; + SDS.qwthl_sec[offset] = qwthl_sec; + SDS.w3[offset] = s*1.0; + } + } + + // Call the C++ implementation. + shoc_assumed_pdf(SDS); + + // Check the result + + // With such a turbulence and scalar profile, this should have + // encouraged cloud everywhere. Verify that this is true. + + // Also verify that output lies within some reasonable bounds. + + // Then verify vertical velocity skewness info + + for(Int s = 0; s < shcol; ++s) { + for(Int n = 0; n < nlevi; ++n) { + const auto offset = n + s * nlevi; + const auto offsets = n + (s+1) * nlevi; + if (s < shcol-1){ + // Verify input w3 is greater in subsequent columns + REQUIRE(SDS.w3[offsets] > SDS.w3[offset]); + } + } + + // Verify output falls within reasonable bounds. For this positive + // vertical velocity skewness test and with the give inputs there + // should be cloud everywhere and all flux terms should be positive. + + for(Int n = 0; n < nlev; ++n) { + const auto offset = n + s * nlev; + const auto offsets = n + (s+1) * nlev; + + REQUIRE( (SDS.shoc_cldfrac[offset] > 0 || SDS.shoc_cldfrac[offset] < 1) ); + REQUIRE(SDS.wqls[offset] > 0); + REQUIRE(SDS.wthv_sec[offset] > 0); + REQUIRE(SDS.shoc_ql2[offset] > 0); + REQUIRE(SDS.shoc_ql[offset] > 0); + + REQUIRE(SDS.wqls[offset] < 0.1); + REQUIRE(SDS.wthv_sec[offset] < wthv_sec_bound); + REQUIRE(SDS.shoc_ql2[offset] < shoc_ql2_bound); + REQUIRE(SDS.shoc_ql[offset] < shoc_ql_bound); + + // Now verify that the relationships in a strongly positive vertical + // velocity flux regime hold true, relative to a symmetric vertical + // velocity regime. + if (s < shcol-1){ + + if (SDS.shoc_cldfrac[offset] < 0.5){ + REQUIRE(SDS.shoc_cldfrac[offsets] < SDS.shoc_cldfrac[offset]); + } + else if (SDS.shoc_cldfrac[offset] > 0.5){ + REQUIRE(SDS.shoc_cldfrac[offsets] > SDS.shoc_cldfrac[offset]); + } + + // In addition, in a positive skewness environment, the following + // should also be true + + // Grid mean liquid water decreased + REQUIRE(SDS.shoc_ql[offsets] < SDS.shoc_ql[offset]); + // liquid water flux increased + REQUIRE(SDS.wqls[offsets] > SDS.wqls[offset]); + // buoyancy flux increased + REQUIRE(SDS.wthv_sec[offsets] > SDS.wthv_sec[offset]); + // liquid water variance increased + REQUIRE(SDS.shoc_ql2[offsets] > SDS.shoc_ql2[offset]); + } + } + } + + // TEST FIVE + // Negative vertical velocity skewness. + + // Using same input as the above test, feed one column with zero skeweness + // and another test with negative vertical velocity skewness and verify + // result is physical. + + // Load input data + for(Int s = 0; s < shcol; ++s) { + for(Int n = 0; n < nlevi; ++n) { + const auto offset = n + s * nlevi; + + SDS.w3[offset] = s*-1.0; + } + } + + // Call the C++ implementation. + shoc_assumed_pdf(SDS); + + // Check the result + + // Verify that output lies within some reasonable bounds. + + // Then verify vertical velocity skewness info + + for(Int s = 0; s < shcol; ++s) { + for(Int n = 0; n < nlevi; ++n) { + const auto offset = n + s * nlevi; + const auto offsets = n + (s+1) * nlevi; + if (s < shcol-1){ + // Verify input w3 is greater in subsequent columns + REQUIRE(SDS.w3[offsets] < SDS.w3[offset]); + } + } + + // Verify output falls within reasonable bounds + // For this negative vertical velocity test some variables + // will be expected to be less than zero. + + for(Int n = 0; n < nlev; ++n) { + const auto offset = n + s * nlev; + const auto offsets = n + (s+1) * nlev; + + REQUIRE( (SDS.shoc_cldfrac[offset] >= 0 || SDS.shoc_cldfrac[offset] < 1) ); + REQUIRE(SDS.shoc_ql2[offset] > 0); + REQUIRE(SDS.shoc_ql[offset] >= 0); + + REQUIRE(std::abs(SDS.wqls[offset] ) < wqls_bound); + REQUIRE(std::abs(SDS.wthv_sec[offset]) < wthv_sec_bound); + REQUIRE(SDS.shoc_ql2[offset] < shoc_ql2_bound); + REQUIRE(SDS.shoc_ql[offset] < shoc_ql_bound); + + // Now verify that the relationships in a strongly negative vertical + // velocity flux regime hold true, relative to a symmetric vertical + // velocity regime. + if (s < shcol-1){ + + if (SDS.shoc_cldfrac[offset] < 0.5){ + REQUIRE(SDS.shoc_cldfrac[offsets] < SDS.shoc_cldfrac[offset]); + } + else if (SDS.shoc_cldfrac[offset] > 0.5){ + REQUIRE(SDS.shoc_cldfrac[offsets] > SDS.shoc_cldfrac[offset]); + } + + // In addition, in a positive skewness environment, the following + // should also be true + + // Grid mean liquid water decreased + REQUIRE(SDS.shoc_ql[offsets] < SDS.shoc_ql[offset]); + // if cloud present, verify liquid water and buoyancy flux is negative + if (SDS.shoc_ql[offsets] > 0){ + REQUIRE(SDS.wqls[offsets] < SDS.wqls[offset]); + REQUIRE(SDS.wqls[offsets] < 0); + + REQUIRE(SDS.wthv_sec[offsets] < SDS.wthv_sec[offset]); + REQUIRE(SDS.wthv_sec[offsets] < 0); + } + + // liquid water variance increased + REQUIRE(SDS.shoc_ql2[offsets] > SDS.shoc_ql2[offset]); + } + } + } + + } + + void run_bfb() + { + auto engine = Base::get_engine(); + + ShocAssumedPdfData SDS_baseline[] = { + // shcol, nlev, nlevi + ShocAssumedPdfData(10, 71, 72), + ShocAssumedPdfData(10, 12, 13), + ShocAssumedPdfData(7, 16, 17), + ShocAssumedPdfData(2, 7, 8), + }; + + // Generate random input data + for (auto& d : SDS_baseline) { + d.randomize(engine, { + {d.thetal, {500, 700}}, + {d.zi_grid, {0, 100}}, + }); + } + + // Create copies of data for use by cxx. Needs to happen before reads so that + // inout data is in original state + ShocAssumedPdfData SDS_cxx[] = { + ShocAssumedPdfData(SDS_baseline[0]), + ShocAssumedPdfData(SDS_baseline[1]), + ShocAssumedPdfData(SDS_baseline[2]), + ShocAssumedPdfData(SDS_baseline[3]), + }; + + static constexpr Int num_runs = sizeof(SDS_baseline) / sizeof(ShocAssumedPdfData); + + // Assume all data is in C layout + + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : SDS_baseline) { + d.read(Base::m_fid); + } + } + + // Get data from cxx + for (auto& d : SDS_cxx) { + shoc_assumed_pdf(d); + } + + // Verify BFB results, all data should be in C layout + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + for (Int i = 0; i < num_runs; ++i) { + ShocAssumedPdfData& d_baseline = SDS_baseline[i]; + ShocAssumedPdfData& d_cxx = SDS_cxx[i]; + for (Int k = 0; k < d_baseline.total(d_baseline.wqls); ++k) { + REQUIRE(d_baseline.shoc_cldfrac[k] == d_cxx.shoc_cldfrac[k]); + REQUIRE(d_baseline.shoc_ql[k] == d_cxx.shoc_ql[k]); + REQUIRE(d_baseline.wqls[k] == d_cxx.wqls[k]); + REQUIRE(d_baseline.wthv_sec[k] == d_cxx.wthv_sec[k]); + REQUIRE(d_baseline.shoc_ql2[k] == d_cxx.shoc_ql2[k]); + } + } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + SDS_cxx[i].write(Base::m_fid); + } + } + } +}; + +} // namespace unit_test +} // namespace shoc +} // namespace scream + +namespace { + +TEST_CASE("shoc_assumed_pdf_property", "shoc") +{ + using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocAssumedPdf; + + TestStruct().run_property(); +} + +TEST_CASE("shoc_assumed_pdf_bfb", "shoc") +{ + using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocAssumedPdf; + + TestStruct().run_bfb(); +} + +} // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_brunt_length_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_brunt_length_tests.cpp index 3018d0e5aa6a..21112e0d0a2f 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_brunt_length_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_brunt_length_tests.cpp @@ -3,7 +3,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -22,9 +22,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestCompBruntShocLength { +struct UnitWrap::UnitTest::TestCompBruntShocLength : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; static constexpr Int nlev = 5; @@ -91,10 +91,7 @@ struct UnitWrap::UnitTest::TestCompBruntShocLength { } // Call the C++ implementation. - SDS.transpose(); - // expects data in fortran layout - compute_brunt_shoc_length_f(SDS.nlev,SDS.nlevi,SDS.shcol,SDS.dz_zt,SDS.thv,SDS.thv_zi,SDS.brunt); - SDS.transpose(); + compute_brunt_shoc_length(SDS); // Check the results for(Int s = 0; s < shcol; ++s) { @@ -124,11 +121,11 @@ struct UnitWrap::UnitTest::TestCompBruntShocLength { } } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - ComputeBruntShocLengthData SDS_f90[] = { + ComputeBruntShocLengthData SDS_baseline[] = { // shcol, nlev, nlevi ComputeBruntShocLengthData(10, 71, 72), ComputeBruntShocLengthData(10, 12, 13), @@ -137,45 +134,49 @@ struct UnitWrap::UnitTest::TestCompBruntShocLength { }; // Generate random input data - for (auto& d : SDS_f90) { + for (auto& d : SDS_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state ComputeBruntShocLengthData SDS_cxx[] = { - ComputeBruntShocLengthData(SDS_f90[0]), - ComputeBruntShocLengthData(SDS_f90[1]), - ComputeBruntShocLengthData(SDS_f90[2]), - ComputeBruntShocLengthData(SDS_f90[3]), + ComputeBruntShocLengthData(SDS_baseline[0]), + ComputeBruntShocLengthData(SDS_baseline[1]), + ComputeBruntShocLengthData(SDS_baseline[2]), + ComputeBruntShocLengthData(SDS_baseline[3]), }; + static constexpr Int num_runs = sizeof(SDS_baseline) / sizeof(ComputeBruntShocLengthData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : SDS_f90) { - // expects data in C layout - compute_brunt_shoc_length(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : SDS_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : SDS_cxx) { - d.transpose(); - // expects data in fortran layout - compute_brunt_shoc_length_f(d.nlev,d.nlevi,d.shcol,d.dz_zt,d.thv,d.thv_zi,d.brunt); - d.transpose(); + compute_brunt_shoc_length(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(SDS_f90) / sizeof(ComputeBruntShocLengthData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - ComputeBruntShocLengthData& d_f90 = SDS_f90[i]; + ComputeBruntShocLengthData& d_baseline = SDS_baseline[i]; ComputeBruntShocLengthData& d_cxx = SDS_cxx[i]; - for (Int k = 0; k < d_f90.total(d_f90.brunt); ++k) { - REQUIRE(d_f90.brunt[k] == d_cxx.brunt[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.brunt); ++k) { + REQUIRE(d_baseline.brunt[k] == d_cxx.brunt[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + SDS_cxx[i].write(Base::m_fid); + } } } }; @@ -190,14 +191,14 @@ TEST_CASE("shoc_brunt_length_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestCompBruntShocLength; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_brunt_length_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestCompBruntShocLength; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_check_length_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_check_length_tests.cpp index bd023e2ed794..404176bfe3ad 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_check_length_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_check_length_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestCheckShocLength { +struct UnitWrap::UnitTest::TestCheckShocLength : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Real maxlen = scream::shoc::Constants::maxlen; static constexpr Real minlen = scream::shoc::Constants::minlen; @@ -76,10 +76,7 @@ struct UnitWrap::UnitTest::TestCheckShocLength { } // Call the C++ implementation. - SDS.transpose(); - // expects data in fortran layout - check_length_scale_shoc_length_f(SDS.nlev,SDS.shcol,SDS.host_dx,SDS.host_dy,SDS.shoc_mix); - SDS.transpose(); + check_length_scale_shoc_length(SDS); // Check the results for(Int s = 0; s < shcol; ++s) { @@ -94,11 +91,11 @@ struct UnitWrap::UnitTest::TestCheckShocLength { } } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - CheckLengthScaleShocLengthData SDS_f90[] = { + CheckLengthScaleShocLengthData SDS_baseline[] = { // shcol, nlev CheckLengthScaleShocLengthData(10, 71), CheckLengthScaleShocLengthData(10, 12), @@ -107,45 +104,49 @@ struct UnitWrap::UnitTest::TestCheckShocLength { }; // Generate random input data - for (auto& d : SDS_f90) { + for (auto& d : SDS_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state CheckLengthScaleShocLengthData SDS_cxx[] = { - CheckLengthScaleShocLengthData(SDS_f90[0]), - CheckLengthScaleShocLengthData(SDS_f90[1]), - CheckLengthScaleShocLengthData(SDS_f90[2]), - CheckLengthScaleShocLengthData(SDS_f90[3]), + CheckLengthScaleShocLengthData(SDS_baseline[0]), + CheckLengthScaleShocLengthData(SDS_baseline[1]), + CheckLengthScaleShocLengthData(SDS_baseline[2]), + CheckLengthScaleShocLengthData(SDS_baseline[3]), }; + static constexpr Int num_runs = sizeof(SDS_baseline) / sizeof(CheckLengthScaleShocLengthData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : SDS_f90) { - // expects data in C layout - check_length_scale_shoc_length(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : SDS_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : SDS_cxx) { - d.transpose(); - // expects data in fortran layout - check_length_scale_shoc_length_f(d.nlev,d.shcol,d.host_dx,d.host_dy,d.shoc_mix); - d.transpose(); + check_length_scale_shoc_length(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(SDS_f90) / sizeof(CheckLengthScaleShocLengthData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - CheckLengthScaleShocLengthData& d_f90 = SDS_f90[i]; + CheckLengthScaleShocLengthData& d_baseline = SDS_baseline[i]; CheckLengthScaleShocLengthData& d_cxx = SDS_cxx[i]; - for (Int k = 0; k < d_f90.total(d_f90.shoc_mix); ++k) { - REQUIRE(d_f90.shoc_mix[k] == d_cxx.shoc_mix[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.shoc_mix); ++k) { + REQUIRE(d_baseline.shoc_mix[k] == d_cxx.shoc_mix[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + SDS_cxx[i].write(Base::m_fid); + } } } }; @@ -160,14 +161,14 @@ TEST_CASE("shoc_check_length_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestCheckShocLength; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_check_length_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestCheckShocLength; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_check_tke_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_check_tke_tests.cpp index 75581e2283ce..5aa189cfb5dc 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_check_tke_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_check_tke_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocCheckTke { +struct UnitWrap::UnitTest::TestShocCheckTke : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Real mintke = scream::shoc::Constants::mintke; static constexpr Int shcol = 2; @@ -53,10 +53,7 @@ struct UnitWrap::UnitTest::TestShocCheckTke { REQUIRE((SDS.shcol > 0 && SDS.nlev > 0)); // Call the C++ implementation. - SDS.transpose(); - // expects data in fortran layout - check_tke_f(SDS.nlev, SDS.shcol, SDS.tke); - SDS.transpose(); + check_tke(SDS); // Check the result against the input values for(Int s = 0; s < shcol; ++s) { @@ -76,11 +73,11 @@ struct UnitWrap::UnitTest::TestShocCheckTke { } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - CheckTkeData SDS_f90[] = { + CheckTkeData SDS_baseline[] = { // shcol, nlev CheckTkeData(10, 71), CheckTkeData(10, 12), @@ -89,45 +86,49 @@ struct UnitWrap::UnitTest::TestShocCheckTke { }; // Generate random input data - for (auto& d : SDS_f90) { + for (auto& d : SDS_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state CheckTkeData SDS_cxx[] = { - CheckTkeData(SDS_f90[0]), - CheckTkeData(SDS_f90[1]), - CheckTkeData(SDS_f90[2]), - CheckTkeData(SDS_f90[3]), + CheckTkeData(SDS_baseline[0]), + CheckTkeData(SDS_baseline[1]), + CheckTkeData(SDS_baseline[2]), + CheckTkeData(SDS_baseline[3]), }; + static constexpr Int num_runs = sizeof(SDS_baseline) / sizeof(CheckTkeData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : SDS_f90) { - // expects data in C layout - check_tke(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : SDS_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : SDS_cxx) { - d.transpose(); - // expects data in fortran layout - check_tke_f(d.nlev, d.shcol, d.tke); - d.transpose(); + check_tke(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(SDS_f90) / sizeof(CheckTkeData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - CheckTkeData& d_f90 = SDS_f90[i]; + CheckTkeData& d_baseline = SDS_baseline[i]; CheckTkeData& d_cxx = SDS_cxx[i]; - for (Int k = 0; k < d_f90.total(d_f90.tke); ++k) { - REQUIRE(d_f90.tke[k] == d_cxx.tke[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.tke); ++k) { + REQUIRE(d_baseline.tke[k] == d_cxx.tke[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + SDS_cxx[i].write(Base::m_fid); + } } } @@ -143,14 +144,14 @@ TEST_CASE("shoc_check_tke_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocCheckTke; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_check_tke_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocCheckTke; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } diff --git a/components/eamxx/src/physics/shoc/tests/shoc_clip_third_moms_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_clip_third_moms_tests.cpp index 28bef1aae259..4f254e65cc59 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_clip_third_moms_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_clip_third_moms_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestClipThirdMoms { +struct UnitWrap::UnitTest::TestClipThirdMoms : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; static constexpr Int nlevi = 5; @@ -83,10 +83,7 @@ struct UnitWrap::UnitTest::TestClipThirdMoms { } // Call the C++ implementation. - SDS.transpose(); - // expects data in fortran layout - clipping_diag_third_shoc_moments_f(SDS.nlevi,SDS.shcol,SDS.w_sec_zi,SDS.w3); - SDS.transpose(); + clipping_diag_third_shoc_moments(SDS); // Check the result // For large values of w3, verify that the result has been reduced @@ -103,11 +100,11 @@ struct UnitWrap::UnitTest::TestClipThirdMoms { } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - ClippingDiagThirdShocMomentsData SDS_f90[] = { + ClippingDiagThirdShocMomentsData SDS_baseline[] = { // shcol, nlevi ClippingDiagThirdShocMomentsData(10, 72), ClippingDiagThirdShocMomentsData(10, 13), @@ -116,45 +113,49 @@ struct UnitWrap::UnitTest::TestClipThirdMoms { }; // Generate random input data - for (auto& d : SDS_f90) { + for (auto& d : SDS_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state ClippingDiagThirdShocMomentsData SDS_cxx[] = { - ClippingDiagThirdShocMomentsData(SDS_f90[0]), - ClippingDiagThirdShocMomentsData(SDS_f90[1]), - ClippingDiagThirdShocMomentsData(SDS_f90[2]), - ClippingDiagThirdShocMomentsData(SDS_f90[3]), + ClippingDiagThirdShocMomentsData(SDS_baseline[0]), + ClippingDiagThirdShocMomentsData(SDS_baseline[1]), + ClippingDiagThirdShocMomentsData(SDS_baseline[2]), + ClippingDiagThirdShocMomentsData(SDS_baseline[3]), }; + static constexpr Int num_runs = sizeof(SDS_baseline) / sizeof(ClippingDiagThirdShocMomentsData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : SDS_f90) { - // expects data in C layout - clipping_diag_third_shoc_moments(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : SDS_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : SDS_cxx) { - d.transpose(); - // expects data in fortran layout - clipping_diag_third_shoc_moments_f(d.nlevi,d.shcol,d.w_sec_zi,d.w3); - d.transpose(); + clipping_diag_third_shoc_moments(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(SDS_f90) / sizeof(ClippingDiagThirdShocMomentsData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - ClippingDiagThirdShocMomentsData& d_f90 = SDS_f90[i]; + ClippingDiagThirdShocMomentsData& d_baseline = SDS_baseline[i]; ClippingDiagThirdShocMomentsData& d_cxx = SDS_cxx[i]; - for (Int k = 0; k < d_f90.total(d_f90.w3); ++k) { - REQUIRE(d_f90.w3[k] == d_cxx.w3[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.w3); ++k) { + REQUIRE(d_baseline.w3[k] == d_cxx.w3[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + SDS_cxx[i].write(Base::m_fid); + } } } }; @@ -169,14 +170,14 @@ TEST_CASE("shoc_clip_third_moms_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestClipThirdMoms; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_clip_third_moms_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestClipThirdMoms; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_compute_diag_third_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_compute_diag_third_tests.cpp index d63bf34bb50e..fc52c64550ba 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_compute_diag_third_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_compute_diag_third_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocCompDiagThird { +struct UnitWrap::UnitTest::TestShocCompDiagThird : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; static constexpr Int nlev = 5; @@ -146,14 +146,7 @@ struct UnitWrap::UnitTest::TestShocCompDiagThird { } // Call the C++ implementation - SDS.transpose(); - // expects data in fortran layout - compute_diag_third_shoc_moment_f(SDS.shcol,SDS.nlev,SDS.nlevi,SDS.w_sec,SDS.thl_sec, - SDS.wthl_sec,SDS.tke,SDS.dz_zt, - SDS.dz_zi,SDS.isotropy_zi, - SDS.brunt_zi,SDS.w_sec_zi,SDS.thetal_zi, - SDS.w3); - SDS.transpose(); + compute_diag_third_shoc_moment(SDS); // Check the result @@ -193,11 +186,11 @@ struct UnitWrap::UnitTest::TestShocCompDiagThird { } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - ComputeDiagThirdShocMomentData SDS_f90[] = { + ComputeDiagThirdShocMomentData SDS_baseline[] = { // shcol, nlev, nlevi ComputeDiagThirdShocMomentData(10, 71, 72), ComputeDiagThirdShocMomentData(10, 12, 13), @@ -206,49 +199,49 @@ struct UnitWrap::UnitTest::TestShocCompDiagThird { }; // Generate random input data - for (auto& d : SDS_f90) { + for (auto& d : SDS_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state ComputeDiagThirdShocMomentData SDS_cxx[] = { - ComputeDiagThirdShocMomentData(SDS_f90[0]), - ComputeDiagThirdShocMomentData(SDS_f90[1]), - ComputeDiagThirdShocMomentData(SDS_f90[2]), - ComputeDiagThirdShocMomentData(SDS_f90[3]), + ComputeDiagThirdShocMomentData(SDS_baseline[0]), + ComputeDiagThirdShocMomentData(SDS_baseline[1]), + ComputeDiagThirdShocMomentData(SDS_baseline[2]), + ComputeDiagThirdShocMomentData(SDS_baseline[3]), }; + static constexpr Int num_runs = sizeof(SDS_baseline) / sizeof(ComputeDiagThirdShocMomentData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : SDS_f90) { - // expects data in C layout - compute_diag_third_shoc_moment(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : SDS_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : SDS_cxx) { - d.transpose(); - // expects data in fortran layout - compute_diag_third_shoc_moment_f(d.shcol,d.nlev,d.nlevi,d.w_sec,d.thl_sec, - d.wthl_sec,d.tke,d.dz_zt, - d.dz_zi,d.isotropy_zi, - d.brunt_zi,d.w_sec_zi,d.thetal_zi, - d.w3); - d.transpose(); + compute_diag_third_shoc_moment(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(SDS_f90) / sizeof(ComputeDiagThirdShocMomentData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - ComputeDiagThirdShocMomentData& d_f90 = SDS_f90[i]; + ComputeDiagThirdShocMomentData& d_baseline = SDS_baseline[i]; ComputeDiagThirdShocMomentData& d_cxx = SDS_cxx[i]; - for (Int k = 0; k < d_f90.total(d_f90.w3); ++k) { - REQUIRE(d_f90.w3[k] == d_cxx.w3[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.w3); ++k) { + REQUIRE(d_baseline.w3[k] == d_cxx.w3[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + SDS_cxx[i].write(Base::m_fid); + } } } }; @@ -263,14 +256,14 @@ TEST_CASE("shoc_comp_diag_third_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocCompDiagThird; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_comp_diag_third_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocCompDiagThird; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_temperature_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_temperature_tests.cpp index 27452cdcf703..0566d12d9320 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_temperature_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_temperature_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestComputeShocTemp { +struct UnitWrap::UnitTest::TestComputeShocTemp : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 1; static constexpr Int nlev = 3; @@ -68,9 +68,7 @@ struct UnitWrap::UnitTest::TestComputeShocTemp { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - compute_shoc_temperature_f(SDS.shcol, SDS.nlev, SDS.thetal, SDS.ql, SDS.inv_exner, SDS.tabs); - SDS.transpose(); // go back to C layout + compute_shoc_temperature(SDS); // Require that absolute temperature is equal to thetal for(Int s = 0; s < shcol; ++s) { @@ -116,9 +114,7 @@ struct UnitWrap::UnitTest::TestComputeShocTemp { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - compute_shoc_temperature_f(SDS.shcol, SDS.nlev, SDS.thetal, SDS.ql, SDS.inv_exner, SDS.tabs); - SDS.transpose(); // go back to C layout + compute_shoc_temperature(SDS); // Require that absolute temperature is greather than thetal for(Int s = 0; s < shcol; ++s) { @@ -177,9 +173,7 @@ struct UnitWrap::UnitTest::TestComputeShocTemp { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - compute_shoc_temperature_f(SDS.shcol, SDS.nlev, SDS.thetal, SDS.ql, SDS.inv_exner, SDS.tabs); - SDS.transpose(); // go back to C layout + compute_shoc_temperature(SDS); // Require that absolute temperature be less than thetal for(Int s = 0; s < shcol; ++s) { @@ -202,11 +196,11 @@ struct UnitWrap::UnitTest::TestComputeShocTemp { } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - ComputeShocTempData f90_data[] = { + ComputeShocTempData baseline_data[] = { // shcol, nlev ComputeShocTempData(10, 71), ComputeShocTempData(10, 12), @@ -215,44 +209,48 @@ struct UnitWrap::UnitTest::TestComputeShocTemp { }; // Generate random input data - for (auto& d : f90_data) { + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state ComputeShocTempData cxx_data[] = { - ComputeShocTempData(f90_data[0]), - ComputeShocTempData(f90_data[1]), - ComputeShocTempData(f90_data[2]), - ComputeShocTempData(f90_data[3]), + ComputeShocTempData(baseline_data[0]), + ComputeShocTempData(baseline_data[1]), + ComputeShocTempData(baseline_data[2]), + ComputeShocTempData(baseline_data[3]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - compute_shoc_temperature(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - compute_shoc_temperature_f(d.shcol, d.nlev, d.thetal, d.ql, d.inv_exner, d.tabs); - d.transpose(); // go back to C layout + compute_shoc_temperature(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(ComputeShocTempData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(ComputeShocTempData); for (Int i = 0; i < num_runs; ++i) { - ComputeShocTempData& d_f90 = f90_data[i]; + ComputeShocTempData& d_baseline = baseline_data[i]; ComputeShocTempData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.tabs); ++k) { - REQUIRE(d_f90.tabs[k] == d_cxx.tabs[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.tabs); ++k) { + REQUIRE(d_baseline.tabs[k] == d_cxx.tabs[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } } }; @@ -267,14 +265,14 @@ TEST_CASE("shoc_compute_shoc_temperature_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestComputeShocTemp; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_compute_shoc_temperature_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestComputeShocTemp; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_vapor_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_vapor_tests.cpp index afd184e18231..1e0d41342045 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_vapor_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_vapor_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "shoc_unit_tests_common.hpp" @@ -14,9 +14,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestComputeShocVapor { +struct UnitWrap::UnitTest::TestComputeShocVapor : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; static constexpr Int nlev = 5; @@ -63,9 +63,7 @@ struct UnitWrap::UnitTest::TestComputeShocVapor { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - compute_shoc_vapor_f(SDS.shcol, SDS.nlev, SDS.qw, SDS.ql, SDS.qv); - SDS.transpose(); // go back to C layout + compute_shoc_vapor(SDS); // Verify the result for(Int s = 0; s < shcol; ++s) { @@ -88,11 +86,11 @@ struct UnitWrap::UnitTest::TestComputeShocVapor { } // run_property - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - ComputeShocVaporData f90_data[] = { + ComputeShocVaporData baseline_data[] = { // shcol, nlev ComputeShocVaporData(10, 71), ComputeShocVaporData(10, 12), @@ -101,44 +99,48 @@ struct UnitWrap::UnitTest::TestComputeShocVapor { }; // Generate random input data - for (auto& d : f90_data) { + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state ComputeShocVaporData cxx_data[] = { - ComputeShocVaporData(f90_data[0]), - ComputeShocVaporData(f90_data[1]), - ComputeShocVaporData(f90_data[2]), - ComputeShocVaporData(f90_data[3]), + ComputeShocVaporData(baseline_data[0]), + ComputeShocVaporData(baseline_data[1]), + ComputeShocVaporData(baseline_data[2]), + ComputeShocVaporData(baseline_data[3]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - compute_shoc_vapor(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - compute_shoc_vapor_f(d.shcol, d.nlev, d.qw, d.ql, d.qv); - d.transpose(); // go back to C layout + compute_shoc_vapor(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(ComputeShocVaporData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(ComputeShocVaporData); for (Int i = 0; i < num_runs; ++i) { - ComputeShocVaporData& d_f90 = f90_data[i]; + ComputeShocVaporData& d_baseline = baseline_data[i]; ComputeShocVaporData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.qv); ++k) { - REQUIRE(d_f90.qv[k] == d_cxx.qv[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.qv); ++k) { + REQUIRE(d_baseline.qv[k] == d_cxx.qv[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } } // run_bfb }; @@ -153,14 +155,14 @@ TEST_CASE("compute_shoc_vapor_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestComputeShocVapor; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("compute_shoc_vapor_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestComputeShocVapor; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_obklen_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_obklen_tests.cpp index 930faf31d6b2..47e4f72ed995 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_obklen_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_obklen_tests.cpp @@ -3,7 +3,7 @@ #include "shoc_unit_tests_common.hpp" #include "physics/share/physics_constants.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocDiagObklen { +struct UnitWrap::UnitTest::TestShocDiagObklen : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 5; @@ -84,12 +84,7 @@ struct UnitWrap::UnitTest::TestShocDiagObklen { } // Call the C++ implementation - SDS.transpose(); - // expects data in fortran layout - shoc_diag_obklen_f(SDS.shcol, SDS.uw_sfc, SDS.vw_sfc, SDS.wthl_sfc, SDS.wqw_sfc, - SDS.thl_sfc, SDS.cldliq_sfc, SDS.qv_sfc, SDS.ustar, SDS.kbfs, - SDS.obklen); - SDS.transpose(); + shoc_diag_obklen(SDS); // Check the result @@ -154,12 +149,7 @@ struct UnitWrap::UnitTest::TestShocDiagObklen { } // Call the C++ implementation - SDS.transpose(); - // expects data in fortran layout - shoc_diag_obklen_f(SDS.shcol, SDS.uw_sfc, SDS.vw_sfc, SDS.wthl_sfc, SDS.wqw_sfc, - SDS.thl_sfc, SDS.cldliq_sfc, SDS.qv_sfc, SDS.ustar, SDS.kbfs, - SDS.obklen); - SDS.transpose(); + shoc_diag_obklen(SDS); // Verify that DIMENSIONLESS obukhov length decreases as columns // increases due to the increasing surface fluxes @@ -172,11 +162,11 @@ struct UnitWrap::UnitTest::TestShocDiagObklen { } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - ShocDiagObklenData SDS_f90[] = { + ShocDiagObklenData SDS_baseline[] = { // shcol ShocDiagObklenData(12), ShocDiagObklenData(10), @@ -185,49 +175,51 @@ struct UnitWrap::UnitTest::TestShocDiagObklen { }; // Generate random input data - for (auto& d : SDS_f90) { + for (auto& d : SDS_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state ShocDiagObklenData SDS_cxx[] = { - ShocDiagObklenData(SDS_f90[0]), - ShocDiagObklenData(SDS_f90[1]), - ShocDiagObklenData(SDS_f90[2]), - ShocDiagObklenData(SDS_f90[3]) + ShocDiagObklenData(SDS_baseline[0]), + ShocDiagObklenData(SDS_baseline[1]), + ShocDiagObklenData(SDS_baseline[2]), + ShocDiagObklenData(SDS_baseline[3]) }; + static constexpr Int num_runs = sizeof(SDS_baseline) / sizeof(ShocDiagObklenData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : SDS_f90) { - // expects data in C layout - shoc_diag_obklen(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : SDS_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : SDS_cxx) { - d.transpose(); - // expects data in fortran layout - shoc_diag_obklen_f(d.shcol, d.uw_sfc, d.vw_sfc, d.wthl_sfc, d.wqw_sfc, - d.thl_sfc, d.cldliq_sfc, d.qv_sfc, d.ustar, d.kbfs, - d.obklen); - d.transpose(); + shoc_diag_obklen(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(SDS_f90) / sizeof(ShocDiagObklenData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - ShocDiagObklenData& d_f90 = SDS_f90[i]; + ShocDiagObklenData& d_baseline = SDS_baseline[i]; ShocDiagObklenData& d_cxx = SDS_cxx[i]; - for (Int s = 0; s < d_f90.shcol; ++s) { - REQUIRE(d_f90.ustar[s] == d_cxx.ustar[s]); - REQUIRE(d_f90.kbfs[s] == d_cxx.kbfs[s]); - REQUIRE(d_f90.obklen[s] == d_cxx.obklen[s]); + for (Int s = 0; s < d_baseline.shcol; ++s) { + REQUIRE(d_baseline.ustar[s] == d_cxx.ustar[s]); + REQUIRE(d_baseline.kbfs[s] == d_cxx.kbfs[s]); + REQUIRE(d_baseline.obklen[s] == d_cxx.obklen[s]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + SDS_cxx[i].write(Base::m_fid); + } } } }; @@ -242,14 +234,14 @@ TEST_CASE("shoc_diag_obklen_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocDiagObklen; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_diag_obklen_length_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocDiagObklen; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_srf_test.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_srf_test.cpp index 25f56139d776..95abfefab984 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_srf_test.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_srf_test.cpp @@ -3,7 +3,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -22,9 +22,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestSecondMomSrf { +struct UnitWrap::UnitTest::TestSecondMomSrf : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { // Property test for the SHOC subroutine: // diag_second_moments_srf @@ -52,7 +52,7 @@ struct UnitWrap::UnitTest::TestSecondMomSrf { } // Call the C++ implementation - shoc_diag_second_moments_srf_f(SDS.shcol, SDS.wthl_sfc, SDS.uw_sfc, SDS.vw_sfc, SDS.ustar2, SDS.wstar); + diag_second_moments_srf(SDS); // Verify the output for (Int s = 0; s < shcol; ++s){ @@ -82,12 +82,12 @@ struct UnitWrap::UnitTest::TestSecondMomSrf { } - static void run_bfb() + void run_bfb() { #if 0 - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - SHOCSecondMomentSrfData mom_srf_data_f90[] = { + SHOCSecondMomentSrfData mom_srf_data_baseline[] = { // shcol SHOCSecondMomentSrfData(36), SHOCSecondMomentSrfData(72), @@ -95,35 +95,40 @@ struct UnitWrap::UnitTest::TestSecondMomSrf { SHOCSecondMomentSrfData(256), }; - for (auto& d : mom_srf_data_f90) { + for (auto& d : mom_srf_data_baseline) { d.randomize(engine, { {d.wthl, {-1, 1}} }); } SHOCSecondMomentSrfData mom_srf_data_cxx[] = { - SHOCSecondMomentSrfData(mom_srf_data_f90[0]), - SHOCSecondMomentSrfData(mom_srf_data_f90[1]), - SHOCSecondMomentSrfData(mom_srf_data_f90[2]), - SHOCSecondMomentSrfData(mom_srf_data_f90[3]), + SHOCSecondMomentSrfData(mom_srf_data_baseline[0]), + SHOCSecondMomentSrfData(mom_srf_data_baseline[1]), + SHOCSecondMomentSrfData(mom_srf_data_baseline[2]), + SHOCSecondMomentSrfData(mom_srf_data_baseline[3]), }; - for (auto& d : mom_srf_data_f90) { + for (auto& d : mom_srf_data_baseline) { // expects data in C layout shoc_diag_second_moments_srf(d); } for (auto& d : mom_srf_data_cxx) { - shoc_diag_second_moments_srf_f(d.shcol, d.wthl_sfc, d.uw_sfc, d.vw_sfc, d.ustar2, d.wstar); + shoc_diag_second_moments_srf(d); } - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(mom_srf_data_f90) / sizeof(SHOCSecondMomentSrfData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(mom_srf_data_baseline) / sizeof(SHOCSecondMomentSrfData); for (Int i = 0; i < num_runs; ++i) { Int shcol = mom_srf_data_cxx[i].shcol; for (Int k = 0; k < shcol; ++k) { - REQUIRE(mom_srf_data_f90[i].ustar2[k] == mom_srf_data_cxx[i].ustar2[k]); - REQUIRE(mom_srf_data_f90[i].wstar[k] == mom_srf_data_cxx[i].wstar[k]); + REQUIRE(mom_srf_data_baseline[i].ustar2[k] == mom_srf_data_cxx[i].ustar2[k]); + REQUIRE(mom_srf_data_baseline[i].wstar[k] == mom_srf_data_cxx[i].wstar[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + cxx_data[i].write(Base::m_fid); + } } #endif } @@ -139,13 +144,13 @@ namespace { TEST_CASE("shoc_second_moments_srf_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestSecondMomSrf; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_second_moments_srf_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestSecondMomSrf; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_ubycond_test.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_ubycond_test.cpp index d4dbd3e7b71f..06f8ccdd45a4 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_ubycond_test.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_ubycond_test.cpp @@ -3,7 +3,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -23,9 +23,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestSecondMomUbycond { +struct UnitWrap::UnitTest::TestSecondMomUbycond : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { // Property test for SHOC subroutine: // diag_second_moments_ubycond @@ -43,8 +43,7 @@ struct UnitWrap::UnitTest::TestSecondMomUbycond { REQUIRE(shcol > 0); // Call the C++ implementation - shoc_diag_second_moments_ubycond_f(SDS.shcol, SDS.thl_sec, SDS.qw_sec, SDS.qwthl_sec, SDS.wthl_sec, - SDS.wqw_sec, SDS.uw_sec, SDS.vw_sec, SDS.wtke_sec); + diag_second_moments_ubycond(SDS); // Verify the result // all output should be zero. @@ -61,9 +60,9 @@ struct UnitWrap::UnitTest::TestSecondMomUbycond { } } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); DiagSecondMomentsUbycondData uby_fortran[] = { // shcol @@ -79,7 +78,7 @@ struct UnitWrap::UnitTest::TestSecondMomUbycond { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state DiagSecondMomentsUbycondData uby_cxx[num_runs] = { DiagSecondMomentsUbycondData(uby_fortran[0]), @@ -88,16 +87,18 @@ struct UnitWrap::UnitTest::TestSecondMomUbycond { DiagSecondMomentsUbycondData(uby_fortran[3]), }; - // Get data from fortran - for (auto& d : uby_fortran) { - diag_second_moments_ubycond(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : uby_fortran) { + d.read(Base::m_fid); + } } for (auto& d : uby_cxx) { - shoc_diag_second_moments_ubycond_f(d.shcol, d.thl_sec, d.qw_sec, d.qwthl_sec, d.wthl_sec, d.wqw_sec, d.uw_sec, d.vw_sec, d.wtke_sec); + diag_second_moments_ubycond(d); } - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { const Int shcol = uby_cxx[i].shcol; for (Int k = 0; k < shcol; ++k) { @@ -111,6 +112,11 @@ struct UnitWrap::UnitTest::TestSecondMomUbycond { REQUIRE(uby_fortran[i].wtke_sec[k] == uby_cxx[i].wtke_sec[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + uby_cxx[i].write(Base::m_fid); + } } } @@ -126,14 +132,14 @@ TEST_CASE("second_mom_uby_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestSecondMomUbycond; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("second_mom_uby_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestSecondMomUbycond; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_lbycond_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_lbycond_tests.cpp index 1f1f3df8e5cc..9680984a4463 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_lbycond_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_lbycond_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "shoc_unit_tests_common.hpp" @@ -14,9 +14,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestDiagSecondMomentsLbycond { +struct UnitWrap::UnitTest::TestDiagSecondMomentsLbycond : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { // Property tests for the SHOC function // diag_second_moments_lbycond @@ -76,10 +76,7 @@ struct UnitWrap::UnitTest::TestDiagSecondMomentsLbycond { } // Call the C++ implementation - diag_second_moments_lbycond_f(SDS.shcol, SDS.wthl_sfc, SDS.wqw_sfc, SDS.uw_sfc, - SDS.vw_sfc, SDS.ustar2, SDS.wstar, SDS.wthl_sec, - SDS.wqw_sec, SDS.uw_sec, SDS.vw_sec, SDS.wtke_sec, - SDS.thl_sec, SDS.qw_sec, SDS.qwthl_sec); + diag_second_moments_lbycond(SDS); // Verify output is as expected for (Int s = 0; s < shcol; ++s){ @@ -118,11 +115,11 @@ struct UnitWrap::UnitTest::TestDiagSecondMomentsLbycond { } // run_property - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - DiagSecondMomentsLbycondData f90_data[] = { + DiagSecondMomentsLbycondData baseline_data[] = { DiagSecondMomentsLbycondData(120), DiagSecondMomentsLbycondData(120), DiagSecondMomentsLbycondData(120), @@ -130,50 +127,55 @@ struct UnitWrap::UnitTest::TestDiagSecondMomentsLbycond { }; // Generate random input data - for (auto& d : f90_data) { + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state DiagSecondMomentsLbycondData cxx_data[] = { - DiagSecondMomentsLbycondData(f90_data[0]), - DiagSecondMomentsLbycondData(f90_data[1]), - DiagSecondMomentsLbycondData(f90_data[2]), - DiagSecondMomentsLbycondData(f90_data[3]), + DiagSecondMomentsLbycondData(baseline_data[0]), + DiagSecondMomentsLbycondData(baseline_data[1]), + DiagSecondMomentsLbycondData(baseline_data[2]), + DiagSecondMomentsLbycondData(baseline_data[3]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - diag_second_moments_lbycond(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - diag_second_moments_lbycond_f(d.shcol, d.wthl_sfc, d.wqw_sfc, d.uw_sfc, d.vw_sfc, d.ustar2, d.wstar, - d.wthl_sec, d.wqw_sec, d.uw_sec, d.vw_sec, d.wtke_sec, d.thl_sec, d.qw_sec, d.qwthl_sec); + diag_second_moments_lbycond(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(DiagSecondMomentsLbycondData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(DiagSecondMomentsLbycondData); for (Int i = 0; i < num_runs; ++i) { - DiagSecondMomentsLbycondData& d_f90 = f90_data[i]; + DiagSecondMomentsLbycondData& d_baseline = baseline_data[i]; DiagSecondMomentsLbycondData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.shcol; ++k) { - REQUIRE(d_f90.wthl_sec[k] == d_cxx.wthl_sec[k]); - REQUIRE(d_f90.wqw_sec[k] == d_cxx.wqw_sec[k]); - REQUIRE(d_f90.uw_sec[k] == d_cxx.uw_sec[k]); - REQUIRE(d_f90.vw_sec[k] == d_cxx.vw_sec[k]); - REQUIRE(d_f90.wtke_sec[k] == d_cxx.wtke_sec[k]); - REQUIRE(d_f90.thl_sec[k] == d_cxx.thl_sec[k]); - REQUIRE(d_f90.qw_sec[k] == d_cxx.qw_sec[k]); - REQUIRE(d_f90.qwthl_sec[k] == d_cxx.qwthl_sec[k]); + for (Int k = 0; k < d_baseline.shcol; ++k) { + REQUIRE(d_baseline.wthl_sec[k] == d_cxx.wthl_sec[k]); + REQUIRE(d_baseline.wqw_sec[k] == d_cxx.wqw_sec[k]); + REQUIRE(d_baseline.uw_sec[k] == d_cxx.uw_sec[k]); + REQUIRE(d_baseline.vw_sec[k] == d_cxx.vw_sec[k]); + REQUIRE(d_baseline.wtke_sec[k] == d_cxx.wtke_sec[k]); + REQUIRE(d_baseline.thl_sec[k] == d_cxx.thl_sec[k]); + REQUIRE(d_baseline.qw_sec[k] == d_cxx.qw_sec[k]); + REQUIRE(d_baseline.qwthl_sec[k] == d_cxx.qwthl_sec[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } } // run_bfb @@ -189,14 +191,14 @@ TEST_CASE("diag_second_moments_lbycond_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestDiagSecondMomentsLbycond; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("diag_second_moments_lbycond_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestDiagSecondMomentsLbycond; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_tests.cpp index f83bfa313f9c..bd5dc1a77b1a 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "shoc_unit_tests_common.hpp" @@ -14,9 +14,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestDiagSecondMoments { +struct UnitWrap::UnitTest::TestDiagSecondMoments : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; static constexpr Int nlev = 5; @@ -163,12 +163,7 @@ struct UnitWrap::UnitTest::TestDiagSecondMoments { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - diag_second_moments_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.thetal, SDS.qw, SDS.u_wind, SDS.v_wind, - SDS.tke, SDS.isotropy, SDS.tkh, SDS.tk, SDS.dz_zi, SDS.zt_grid, SDS.zi_grid, SDS.shoc_mix, - SDS.thl_sec, SDS.qw_sec, SDS.wthl_sec, SDS.wqw_sec, SDS.qwthl_sec, SDS.uw_sec, - SDS.vw_sec, SDS.wtke_sec, SDS.w_sec); - SDS.transpose(); // go back to C layout + diag_second_moments(SDS); // Verify output makes sense for(Int s = 0; s < shcol; ++s) { @@ -254,11 +249,11 @@ struct UnitWrap::UnitTest::TestDiagSecondMoments { } // run_property - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - DiagSecondMomentsData f90_data[] = { + DiagSecondMomentsData baseline_data[] = { DiagSecondMomentsData(36, 72, 73), DiagSecondMomentsData(72, 72, 73), DiagSecondMomentsData(128, 72, 73), @@ -266,57 +261,58 @@ struct UnitWrap::UnitTest::TestDiagSecondMoments { }; // Generate random input data - for (auto& d : f90_data) { + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state DiagSecondMomentsData cxx_data[] = { - DiagSecondMomentsData(f90_data[0]), - DiagSecondMomentsData(f90_data[1]), - DiagSecondMomentsData(f90_data[2]), - DiagSecondMomentsData(f90_data[3]), + DiagSecondMomentsData(baseline_data[0]), + DiagSecondMomentsData(baseline_data[1]), + DiagSecondMomentsData(baseline_data[2]), + DiagSecondMomentsData(baseline_data[3]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - diag_second_moments(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - diag_second_moments_f(d.shcol, d.nlev, d.nlevi, d.thetal, d.qw, d.u_wind, d.v_wind, - d.tke, d.isotropy, d.tkh, d.tk, d.dz_zi, d.zt_grid, d.zi_grid, d.shoc_mix, - d.thl_sec, d.qw_sec, d.wthl_sec, d.wqw_sec, d.qwthl_sec, d.uw_sec, - d.vw_sec, d.wtke_sec, d.w_sec); - d.transpose(); // go back to C layout + diag_second_moments(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(DiagSecondMomentsData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(DiagSecondMomentsData); for (Int i = 0; i < num_runs; ++i) { - DiagSecondMomentsData& d_f90 = f90_data[i]; + DiagSecondMomentsData& d_baseline = baseline_data[i]; DiagSecondMomentsData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.w_sec); ++k) { - REQUIRE(d_f90.w_sec[k] == d_cxx.w_sec[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.w_sec); ++k) { + REQUIRE(d_baseline.w_sec[k] == d_cxx.w_sec[k]); } - for (Int k = 0; k < d_f90.total(d_f90.thl_sec); ++k) { - REQUIRE(d_f90.thl_sec[k] == d_cxx.thl_sec[k]); - REQUIRE(d_f90.qw_sec[k] == d_cxx.qw_sec[k]); - REQUIRE(d_f90.wthl_sec[k] == d_cxx.wthl_sec[k]); - REQUIRE(d_f90.wqw_sec[k] == d_cxx.wqw_sec[k]); - REQUIRE(d_f90.qwthl_sec[k] == d_cxx.qwthl_sec[k]); - REQUIRE(d_f90.uw_sec[k] == d_cxx.uw_sec[k]); - REQUIRE(d_f90.vw_sec[k] == d_cxx.vw_sec[k]); - REQUIRE(d_f90.wtke_sec[k] == d_cxx.wtke_sec[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.thl_sec); ++k) { + REQUIRE(d_baseline.thl_sec[k] == d_cxx.thl_sec[k]); + REQUIRE(d_baseline.qw_sec[k] == d_cxx.qw_sec[k]); + REQUIRE(d_baseline.wthl_sec[k] == d_cxx.wthl_sec[k]); + REQUIRE(d_baseline.wqw_sec[k] == d_cxx.wqw_sec[k]); + REQUIRE(d_baseline.qwthl_sec[k] == d_cxx.qwthl_sec[k]); + REQUIRE(d_baseline.uw_sec[k] == d_cxx.uw_sec[k]); + REQUIRE(d_baseline.vw_sec[k] == d_cxx.vw_sec[k]); + REQUIRE(d_baseline.wtke_sec[k] == d_cxx.wtke_sec[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } } // run_bfb @@ -332,14 +328,14 @@ TEST_CASE("diag_second_moments_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestDiagSecondMoments; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("diag_second_moments_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestDiagSecondMoments; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_shoc_moments_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_shoc_moments_tests.cpp index 0385335c808b..45964c6b1d57 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_shoc_moments_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_shoc_moments_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "shoc_unit_tests_common.hpp" @@ -14,9 +14,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestDiagSecondShocMoments { +struct UnitWrap::UnitTest::TestDiagSecondShocMoments : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; @@ -161,12 +161,7 @@ struct UnitWrap::UnitTest::TestDiagSecondShocMoments { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - diag_second_shoc_moments_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.thetal, SDS.qw, SDS.u_wind, SDS.v_wind, SDS.tke, SDS.isotropy, - SDS.tkh, SDS.tk, SDS.dz_zi, SDS.zt_grid, SDS.zi_grid, SDS.shoc_mix, SDS.wthl_sfc, SDS.wqw_sfc, - SDS.uw_sfc, SDS.vw_sfc, SDS.thl_sec, SDS.qw_sec, SDS.wthl_sec, SDS.wqw_sec, SDS.qwthl_sec, - SDS.uw_sec, SDS.vw_sec, SDS.wtke_sec, SDS.w_sec); - SDS.transpose(); // go back to C layout + diag_second_shoc_moments(SDS); // Verify output makes sense for(Int s = 0; s < shcol; ++s) { @@ -267,11 +262,11 @@ struct UnitWrap::UnitTest::TestDiagSecondShocMoments { } // run_property - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - DiagSecondShocMomentsData f90_data[] = { + DiagSecondShocMomentsData baseline_data[] = { DiagSecondShocMomentsData(36, 72, 73), DiagSecondShocMomentsData(72, 72, 73), DiagSecondShocMomentsData(128, 72, 73), @@ -279,56 +274,58 @@ struct UnitWrap::UnitTest::TestDiagSecondShocMoments { }; // Generate random input data - for (auto& d : f90_data) { + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state DiagSecondShocMomentsData cxx_data[] = { - DiagSecondShocMomentsData(f90_data[0]), - DiagSecondShocMomentsData(f90_data[1]), - DiagSecondShocMomentsData(f90_data[2]), - DiagSecondShocMomentsData(f90_data[3]), + DiagSecondShocMomentsData(baseline_data[0]), + DiagSecondShocMomentsData(baseline_data[1]), + DiagSecondShocMomentsData(baseline_data[2]), + DiagSecondShocMomentsData(baseline_data[3]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - diag_second_shoc_moments(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - diag_second_shoc_moments_f(d.shcol, d.nlev, d.nlevi, d.thetal, d.qw, d.u_wind, d.v_wind, d.tke, d.isotropy, - d.tkh, d.tk, d.dz_zi, d.zt_grid, d.zi_grid, d.shoc_mix, d.wthl_sfc, d.wqw_sfc, d.uw_sfc, d.vw_sfc, d.thl_sec, - d.qw_sec, d.wthl_sec, d.wqw_sec, d.qwthl_sec, d.uw_sec, d.vw_sec, d.wtke_sec, d.w_sec); - d.transpose(); // go back to C layout + diag_second_shoc_moments(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(DiagSecondShocMomentsData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(DiagSecondShocMomentsData); for (Int i = 0; i < num_runs; ++i) { - DiagSecondShocMomentsData& d_f90 = f90_data[i]; + DiagSecondShocMomentsData& d_baseline = baseline_data[i]; DiagSecondShocMomentsData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.w_sec); ++k) { - REQUIRE(d_f90.w_sec[k] == d_cxx.w_sec[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.w_sec); ++k) { + REQUIRE(d_baseline.w_sec[k] == d_cxx.w_sec[k]); } - for (Int k = 0; k < d_f90.total(d_f90.thl_sec); ++k) { - REQUIRE(d_f90.thl_sec[k] == d_cxx.thl_sec[k]); - REQUIRE(d_f90.qw_sec[k] == d_cxx.qw_sec[k]); - REQUIRE(d_f90.wthl_sec[k] == d_cxx.wthl_sec[k]); - REQUIRE(d_f90.wqw_sec[k] == d_cxx.wqw_sec[k]); - REQUIRE(d_f90.qwthl_sec[k] == d_cxx.qwthl_sec[k]); - REQUIRE(d_f90.uw_sec[k] == d_cxx.uw_sec[k]); - REQUIRE(d_f90.vw_sec[k] == d_cxx.vw_sec[k]); - REQUIRE(d_f90.wtke_sec[k] == d_cxx.wtke_sec[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.thl_sec); ++k) { + REQUIRE(d_baseline.thl_sec[k] == d_cxx.thl_sec[k]); + REQUIRE(d_baseline.qw_sec[k] == d_cxx.qw_sec[k]); + REQUIRE(d_baseline.wthl_sec[k] == d_cxx.wthl_sec[k]); + REQUIRE(d_baseline.wqw_sec[k] == d_cxx.wqw_sec[k]); + REQUIRE(d_baseline.qwthl_sec[k] == d_cxx.qwthl_sec[k]); + REQUIRE(d_baseline.uw_sec[k] == d_cxx.uw_sec[k]); + REQUIRE(d_baseline.vw_sec[k] == d_cxx.vw_sec[k]); + REQUIRE(d_baseline.wtke_sec[k] == d_cxx.wtke_sec[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } } // run_bfb @@ -344,14 +341,14 @@ TEST_CASE("diag_second_shoc_moments_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestDiagSecondShocMoments; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("diag_second_shoc_moments_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestDiagSecondShocMoments; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_third_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_third_tests.cpp index 82b4d88c0f70..3f1bfd557f02 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_third_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_third_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocDiagThird { +struct UnitWrap::UnitTest::TestShocDiagThird : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; static constexpr Int nlev = 5; @@ -143,13 +143,7 @@ struct UnitWrap::UnitTest::TestShocDiagThird { } // Call the C++ implementation - SDS.transpose(); - // expects data in fortran layout - diag_third_shoc_moments_f(SDS.shcol,SDS.nlev,SDS.nlevi,SDS.w_sec,SDS.thl_sec, - SDS.wthl_sec,SDS.isotropy,SDS.brunt,SDS.thetal, - SDS.tke,SDS.dz_zt,SDS.dz_zi,SDS.zt_grid,SDS.zi_grid, - SDS.w3); - SDS.transpose(); + diag_third_shoc_moments(SDS); // Check to make sure there is at least one // positive w3 value for convective boundary layer @@ -193,13 +187,7 @@ struct UnitWrap::UnitTest::TestShocDiagThird { } // Call the C++ implementation - SDS.transpose(); - // expects data in fortran layout - diag_third_shoc_moments_f(SDS.shcol,SDS.nlev,SDS.nlevi,SDS.w_sec,SDS.thl_sec, - SDS.wthl_sec,SDS.isotropy,SDS.brunt,SDS.thetal, - SDS.tke,SDS.dz_zt,SDS.dz_zi,SDS.zt_grid,SDS.zi_grid, - SDS.w3); - SDS.transpose(); + diag_third_shoc_moments(SDS); // Verify that new result is greater or equal in magnitude // that the result from test one @@ -214,11 +202,11 @@ struct UnitWrap::UnitTest::TestShocDiagThird { } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - DiagThirdShocMomentsData SDS_f90[] = { + DiagThirdShocMomentsData SDS_baseline[] = { // shcol, nlev, nlevi DiagThirdShocMomentsData(10, 71, 72), DiagThirdShocMomentsData(10, 12, 13), @@ -227,48 +215,49 @@ struct UnitWrap::UnitTest::TestShocDiagThird { }; // Generate random input data - for (auto& d : SDS_f90) { + for (auto& d : SDS_baseline) { d.randomize(engine, {{d.thetal, {300, 301}}}); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state DiagThirdShocMomentsData SDS_cxx[] = { - DiagThirdShocMomentsData(SDS_f90[0]), - DiagThirdShocMomentsData(SDS_f90[1]), - DiagThirdShocMomentsData(SDS_f90[2]), - DiagThirdShocMomentsData(SDS_f90[3]), + DiagThirdShocMomentsData(SDS_baseline[0]), + DiagThirdShocMomentsData(SDS_baseline[1]), + DiagThirdShocMomentsData(SDS_baseline[2]), + DiagThirdShocMomentsData(SDS_baseline[3]), }; + static constexpr Int num_runs = sizeof(SDS_baseline) / sizeof(DiagThirdShocMomentsData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : SDS_f90) { - // expects data in C layout - diag_third_shoc_moments(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : SDS_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : SDS_cxx) { - d.transpose(); - // expects data in fortran layout - diag_third_shoc_moments_f(d.shcol,d.nlev,d.nlevi,d.w_sec,d.thl_sec, - d.wthl_sec,d.isotropy,d.brunt,d.thetal, - d.tke,d.dz_zt,d.dz_zi,d.zt_grid,d.zi_grid, - d.w3); - d.transpose(); + diag_third_shoc_moments(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(SDS_f90) / sizeof(DiagThirdShocMomentsData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - DiagThirdShocMomentsData& d_f90 = SDS_f90[i]; + DiagThirdShocMomentsData& d_baseline = SDS_baseline[i]; DiagThirdShocMomentsData& d_cxx = SDS_cxx[i]; - for (Int k = 0; k < d_f90.total(d_f90.w3); ++k) { - REQUIRE(d_f90.w3[k] == d_cxx.w3[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.w3); ++k) { + REQUIRE(d_baseline.w3[k] == d_cxx.w3[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + SDS_cxx[i].write(Base::m_fid); + } } } }; @@ -283,14 +272,14 @@ TEST_CASE("shoc_diag_third_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocDiagThird; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_diag_third_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocDiagThird; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_eddy_diffusivities_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_eddy_diffusivities_tests.cpp index 0a96e5adc629..3d33d55640ca 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_eddy_diffusivities_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_eddy_diffusivities_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -22,9 +22,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocEddyDiff { +struct UnitWrap::UnitTest::TestShocEddyDiff : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; static constexpr Int nlev = 1; @@ -98,10 +98,7 @@ struct UnitWrap::UnitTest::TestShocEddyDiff { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - eddy_diffusivities_f(SDS.nlev, SDS.shcol, SDS.pblh, SDS.zt_grid, SDS.tabs, SDS.shoc_mix, - SDS.sterm_zt, SDS.isotropy, SDS.tke, SDS.tkh, SDS.tk); - SDS.transpose(); // go back to C layout + eddy_diffusivities(SDS); // Check to make sure the answers in the columns are different for(Int s = 0; s < shcol-1; ++s) { @@ -168,10 +165,7 @@ struct UnitWrap::UnitTest::TestShocEddyDiff { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - eddy_diffusivities_f(SDS.nlev, SDS.shcol, SDS.pblh, SDS.zt_grid, SDS.tabs, SDS.shoc_mix, - SDS.sterm_zt, SDS.isotropy, SDS.tke, SDS.tkh, SDS.tk); - SDS.transpose(); // go back to C layout + eddy_diffusivities(SDS); // Check to make sure the answers in the columns are larger // when the length scale and shear term are larger @@ -241,10 +235,7 @@ struct UnitWrap::UnitTest::TestShocEddyDiff { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - eddy_diffusivities_f(SDS.nlev, SDS.shcol, SDS.pblh, SDS.zt_grid, SDS.tabs, SDS.shoc_mix, - SDS.sterm_zt, SDS.isotropy, SDS.tke, SDS.tkh, SDS.tk); - SDS.transpose(); // go back to C layout + eddy_diffusivities(SDS); // Check to make sure the diffusivities are smaller // in the columns where isotropy and tke are smaller @@ -263,11 +254,11 @@ struct UnitWrap::UnitTest::TestShocEddyDiff { } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - EddyDiffusivitiesData f90_data[] = { + EddyDiffusivitiesData baseline_data[] = { EddyDiffusivitiesData(10, 71), EddyDiffusivitiesData(10, 12), EddyDiffusivitiesData(7, 16), @@ -275,46 +266,50 @@ struct UnitWrap::UnitTest::TestShocEddyDiff { }; // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { + // Alternatively, you can use the baseline_data construtors/initializer lists to hardcode data + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state EddyDiffusivitiesData cxx_data[] = { - EddyDiffusivitiesData(f90_data[0]), - EddyDiffusivitiesData(f90_data[1]), - EddyDiffusivitiesData(f90_data[2]), - EddyDiffusivitiesData(f90_data[3]), + EddyDiffusivitiesData(baseline_data[0]), + EddyDiffusivitiesData(baseline_data[1]), + EddyDiffusivitiesData(baseline_data[2]), + EddyDiffusivitiesData(baseline_data[3]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - eddy_diffusivities(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - eddy_diffusivities_f(d.nlev, d.shcol, d.pblh, d.zt_grid, d.tabs, d.shoc_mix, d.sterm_zt, d.isotropy, d.tke, d.tkh, d.tk); - d.transpose(); // go back to C layout + eddy_diffusivities(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(EddyDiffusivitiesData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(EddyDiffusivitiesData); for (Int i = 0; i < num_runs; ++i) { - EddyDiffusivitiesData& d_f90 = f90_data[i]; + EddyDiffusivitiesData& d_baseline = baseline_data[i]; EddyDiffusivitiesData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.tkh); ++k) { - REQUIRE(d_f90.tkh[k] == d_cxx.tkh[k]); - REQUIRE(d_f90.tk[k] == d_cxx.tk[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.tkh); ++k) { + REQUIRE(d_baseline.tkh[k] == d_cxx.tkh[k]); + REQUIRE(d_baseline.tk[k] == d_cxx.tk[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } } // run_bfb }; @@ -329,14 +324,14 @@ TEST_CASE("shoc_tke_eddy_diffusivities_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocEddyDiff; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_tke_eddy_diffusivities_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocEddyDiff; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_energy_dse_fixer_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_energy_dse_fixer_tests.cpp deleted file mode 100644 index 9b91a7ef0b2f..000000000000 --- a/components/eamxx/src/physics/shoc/tests/shoc_energy_dse_fixer_tests.cpp +++ /dev/null @@ -1,137 +0,0 @@ -#include "catch2/catch.hpp" - -#include "shoc_unit_tests_common.hpp" - -#include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" -#include "physics/share/physics_constants.hpp" -#include "share/scream_types.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/util/ekat_arch.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" - -#include -#include -#include -#include - -namespace scream { -namespace shoc { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestShocEnergyDseFixer { - - static void run_property() - { - static constexpr Int shcol = 6; - static constexpr Int nlev = 5; - - // Tests for the SHOC function - // shoc_energy_dse_fixer - - // TEST - // For columns that are identical EXCEPT for the shoctop indicee, - // verify that given a positive value of energy imbalance that - // columns with a higher SHOC top were subject to more energy removal. - - // Host model dry static energy [J kg-1] - static constexpr Real host_dse_input[nlev] = {350e3, 325e3, 315e3, 310e3, 300e3}; - - // Energy disbalance. For this test we assume all columns have - // the same disbalance magnitude. - static constexpr Real se_dis = 0.1; - - // level indicee of SHOC top layer - static constexpr Int shoctop[shcol] = {5, 3, 1, 2, 4, 4}; - - // Initialize data structure for bridging to F90 - ShocEnergyDseFixerData SDS(shcol, nlev); - - // Test that the inputs are reasonable. - // for this test we need exactly six columns - REQUIRE( (SDS.shcol == 6 && SDS.nlev == nlev) ); - - // Fill in test data on zt_grid. - for(Int s = 0; s < shcol; ++s) { - SDS.shoctop[s] = shoctop[s]; - SDS.se_dis[s] = se_dis; - for(Int n = 0; n < nlev; ++n) { - const auto offset = n + s * nlev; - - SDS.host_dse[offset] = host_dse_input[n]; - } - } - - // Check that the inputs make sense - - for(Int s = 0; s < shcol; ++s) { - // For this test we WANT se_dis > 0 - REQUIRE(SDS.se_dis[s] > 0.0); - REQUIRE(SDS.shoctop[s] >= 1); - REQUIRE(SDS.shoctop[s] <= nlev); - for (Int n = 0; n < nlev; ++n){ - const auto offset = n + s * nlev; - - REQUIRE(SDS.host_dse[offset] > 0.0); - } - } - - // Call the fortran implementation - shoc_energy_dse_fixer(SDS); - - // Check the results - Real temp_sum[shcol]; - for(Int s = 0; s < shcol; ++s) { - temp_sum[s] = 0.0; - for(Int n = 0; n < nlev; ++n) { - const auto offset = n + s * nlev; - temp_sum[s] += SDS.host_dse[offset]; - } - } - - // Verify that as shoctop values get lower that the - // summation of temperatures also gets lower. This is proportionally - // to the amount of energy we expect to be removed from a column. - for (Int s = 0; s < shcol-1; ++s) { - if (shoctop[s] < shoctop[s+1]){ - REQUIRE(temp_sum[s] < temp_sum[s+1]); - } - else if (shoctop[s] > shoctop[s+1]){ - REQUIRE(temp_sum[s] > temp_sum[s+1]); - } - else{ - REQUIRE(temp_sum[s] == temp_sum[s+1]); - } - } - - } - - static void run_bfb() - { - // TODO - } -}; - -} // namespace unit_test -} // namespace shoc -} // namespace scream - -namespace { - -TEST_CASE("shoc_energy_dse_fixer_property", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocEnergyDseFixer; - - TestStruct::run_property(); -} - -TEST_CASE("shoc_energy_dse_fixer_bfb", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocEnergyDseFixer; - - TestStruct::run_bfb(); -} - -} // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_energy_fixer_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_energy_fixer_tests.cpp index dfb37104f018..89deaf7ad4bb 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_energy_fixer_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_energy_fixer_tests.cpp @@ -3,7 +3,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "shoc_constants.hpp" #include "share/scream_types.hpp" @@ -23,9 +23,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocEnergyFixer { +struct UnitWrap::UnitTest::TestShocEnergyFixer : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Real gravit = scream::physics::Constants::gravit; static constexpr Real Cpair = scream::physics::Constants::Cpair; @@ -158,14 +158,7 @@ struct UnitWrap::UnitTest::TestShocEnergyFixer { } // Call the C++ implementation - SDS.transpose(); - // expects data in fortran layout - shoc_energy_fixer_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.dtime, SDS.nadv, - SDS.zt_grid, SDS.zi_grid, SDS.se_b, SDS.ke_b, SDS.wv_b, - SDS.wl_b, SDS.se_a, SDS.ke_a, SDS.wv_a, SDS.wl_a, SDS.wthl_sfc, - SDS.wqw_sfc, SDS.rho_zt, SDS.tke, SDS.pint, - SDS.host_dse); - SDS.transpose(); + shoc_energy_fixer(SDS); // Check test // Verify that the dry static energy has not changed if surface @@ -239,14 +232,7 @@ struct UnitWrap::UnitTest::TestShocEnergyFixer { } // Call the C++ implementation - SDS.transpose(); - // expects data in fortran layout - shoc_energy_fixer_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.dtime, SDS.nadv, - SDS.zt_grid, SDS.zi_grid, SDS.se_b, SDS.ke_b, SDS.wv_b, - SDS.wl_b, SDS.se_a, SDS.ke_a, SDS.wv_a, SDS.wl_a, SDS.wthl_sfc, - SDS.wqw_sfc, SDS.rho_zt, SDS.tke, SDS.pint, - SDS.host_dse); - SDS.transpose(); + shoc_energy_fixer(SDS); // Verify the result for(Int s = 0; s < shcol; ++s) { @@ -275,11 +261,11 @@ struct UnitWrap::UnitTest::TestShocEnergyFixer { } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - ShocEnergyFixerData SDS_f90[] = { + ShocEnergyFixerData SDS_baseline[] = { // shcol, nlev, nlevi, dtime, nadv ShocEnergyFixerData(10, 71, 72, 300, 2), ShocEnergyFixerData(10, 12, 13, 100, 10), @@ -288,49 +274,49 @@ struct UnitWrap::UnitTest::TestShocEnergyFixer { }; // Generate random input data - for (auto& d : SDS_f90) { + for (auto& d : SDS_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state ShocEnergyFixerData SDS_cxx[] = { - ShocEnergyFixerData(SDS_f90[0]), - ShocEnergyFixerData(SDS_f90[1]), - ShocEnergyFixerData(SDS_f90[2]), - ShocEnergyFixerData(SDS_f90[3]), + ShocEnergyFixerData(SDS_baseline[0]), + ShocEnergyFixerData(SDS_baseline[1]), + ShocEnergyFixerData(SDS_baseline[2]), + ShocEnergyFixerData(SDS_baseline[3]), }; + static constexpr Int num_runs = sizeof(SDS_baseline) / sizeof(ShocEnergyFixerData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : SDS_f90) { - // expects data in C layout - shoc_energy_fixer(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : SDS_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : SDS_cxx) { - d.transpose(); - // expects data in fortran layout - shoc_energy_fixer_f(d.shcol, d.nlev, d.nlevi, d.dtime, d.nadv, - d.zt_grid, d.zi_grid, d.se_b, d.ke_b, d.wv_b, - d.wl_b, d.se_a, d.ke_a, d.wv_a, d.wl_a, d.wthl_sfc, - d.wqw_sfc, d.rho_zt, d.tke, d.pint, - d.host_dse); - d.transpose(); + shoc_energy_fixer(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(SDS_f90) / sizeof(ShocEnergyFixerData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - ShocEnergyFixerData& d_f90 = SDS_f90[i]; + ShocEnergyFixerData& d_baseline = SDS_baseline[i]; ShocEnergyFixerData& d_cxx = SDS_cxx[i]; - for (Int k = 0; k < d_f90.total(d_f90.host_dse); ++k) { - REQUIRE(d_f90.host_dse[k] == d_cxx.host_dse[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.host_dse); ++k) { + REQUIRE(d_baseline.host_dse[k] == d_cxx.host_dse[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + SDS_cxx[i].write(Base::m_fid); + } } } }; @@ -345,14 +331,14 @@ TEST_CASE("shoc_energy_fixer_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocEnergyFixer; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_energy_fixer_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocEnergyFixer; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_energy_integral_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_energy_integral_tests.cpp index 8142da9371da..e26ec03139f7 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_energy_integral_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_energy_integral_tests.cpp @@ -3,7 +3,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -22,9 +22,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocEnergyInt { +struct UnitWrap::UnitTest::TestShocEnergyInt : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; static constexpr Int nlev = 5; @@ -108,12 +108,7 @@ struct UnitWrap::UnitTest::TestShocEnergyInt { } // Call the C++ implementation - SDS.transpose(); - // expects data in fortran layout - shoc_energy_integrals_f(SDS.shcol, SDS.nlev, SDS.host_dse, SDS.pdel, - SDS.rtm, SDS.rcm, SDS.u_wind, SDS.v_wind, - SDS.se_int, SDS.ke_int, SDS.wv_int, SDS.wl_int); - SDS.transpose(); + shoc_energy_integrals(SDS); // Check test for(Int s = 0; s < shcol; ++s) { @@ -132,11 +127,11 @@ struct UnitWrap::UnitTest::TestShocEnergyInt { } } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - ShocEnergyIntegralsData SDS_f90[] = { + ShocEnergyIntegralsData SDS_baseline[] = { // shcol, nlev ShocEnergyIntegralsData(10, 71), ShocEnergyIntegralsData(10, 12), @@ -145,50 +140,52 @@ struct UnitWrap::UnitTest::TestShocEnergyInt { }; // Generate random input data - for (auto& d : SDS_f90) { + for (auto& d : SDS_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state ShocEnergyIntegralsData SDS_cxx[] = { - ShocEnergyIntegralsData(SDS_f90[0]), - ShocEnergyIntegralsData(SDS_f90[1]), - ShocEnergyIntegralsData(SDS_f90[2]), - ShocEnergyIntegralsData(SDS_f90[3]), + ShocEnergyIntegralsData(SDS_baseline[0]), + ShocEnergyIntegralsData(SDS_baseline[1]), + ShocEnergyIntegralsData(SDS_baseline[2]), + ShocEnergyIntegralsData(SDS_baseline[3]), }; + static constexpr Int num_runs = sizeof(SDS_baseline) / sizeof(ShocEnergyIntegralsData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : SDS_f90) { - // expects data in C layout - shoc_energy_integrals(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : SDS_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : SDS_cxx) { - d.transpose(); - // expects data in fortran layout - shoc_energy_integrals_f(d.shcol, d.nlev, d.host_dse, d.pdel, - d.rtm, d.rcm, d.u_wind, d.v_wind, - d.se_int, d.ke_int, d.wv_int, d.wl_int); - d.transpose(); + shoc_energy_integrals(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(SDS_f90) / sizeof(ShocEnergyIntegralsData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - ShocEnergyIntegralsData& d_f90 = SDS_f90[i]; + ShocEnergyIntegralsData& d_baseline = SDS_baseline[i]; ShocEnergyIntegralsData& d_cxx = SDS_cxx[i]; - for (Int c = 0; c < d_f90.shcol; ++c) { - REQUIRE(d_f90.se_int[c] == d_cxx.se_int[c]); - REQUIRE(d_f90.ke_int[c] == d_cxx.ke_int[c]); - REQUIRE(d_f90.wv_int[c] == d_cxx.wv_int[c]); - REQUIRE(d_f90.wl_int[c] == d_cxx.wl_int[c]); + for (Int c = 0; c < d_baseline.shcol; ++c) { + REQUIRE(d_baseline.se_int[c] == d_cxx.se_int[c]); + REQUIRE(d_baseline.ke_int[c] == d_cxx.ke_int[c]); + REQUIRE(d_baseline.wv_int[c] == d_cxx.wv_int[c]); + REQUIRE(d_baseline.wl_int[c] == d_cxx.wl_int[c]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + SDS_cxx[i].write(Base::m_fid); + } } } }; @@ -203,14 +200,14 @@ TEST_CASE("shoc_energy_integrals_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocEnergyInt; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_energy_integrals_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocEnergyInt; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_energy_threshold_fixer_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_energy_threshold_fixer_tests.cpp deleted file mode 100644 index 19743c391897..000000000000 --- a/components/eamxx/src/physics/shoc/tests/shoc_energy_threshold_fixer_tests.cpp +++ /dev/null @@ -1,139 +0,0 @@ -#include "catch2/catch.hpp" - -#include "shoc_unit_tests_common.hpp" - -#include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" -#include "physics/share/physics_constants.hpp" -#include "share/scream_types.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/util/ekat_arch.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" - -#include -#include -#include -#include - -namespace scream { -namespace shoc { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestShocEnergyThreshFixer { - - static void run_property() - { - static constexpr Real mintke = scream::shoc::Constants::mintke; - static constexpr Int shcol = 2; - static constexpr Int nlev = 5; - static constexpr auto nlevi = nlev + 1; - - // Tests for the SHOC function - // shoc_energy_threshold_fixer - - // TEST ONE - // Set up a reasonable profile verify results are as expected - - // Host model TKE [m2/s2] - Real tke_input[nlev] = {mintke, mintke, 0.01, 0.4, 0.5}; - // Pressure at interface [Pa] - Real pint[nlevi] = {500e2, 600e2, 700e2, 800e2, 900e2, 1000e2}; - - // Integrated total energy after SHOC. - static constexpr Real te_a = 100; - // Integrated total energy before SHOC - static constexpr Real te_b = 110; - - // convert pressure to Pa - for(Int n = 0; n < nlevi; ++n) { - pint[n] = pint[n]; - } - - // Initialize data structure for bridging to F90 - ShocEnergyThresholdFixerData SDS(shcol, nlev, nlevi); - - // Test that the inputs are reasonable. - REQUIRE( (SDS.shcol == shcol && SDS.nlev == nlev && SDS.nlevi == nlevi) ); - REQUIRE(SDS.shcol > 1); - REQUIRE(nlev+1 == nlevi); - - // Fill in test data on zt_grid. - for(Int s = 0; s < shcol; ++s) { - SDS.te_a[s] = te_a; - SDS.te_b[s] = te_b; - for(Int n = 0; n < nlev; ++n) { - const auto offset = n + s * nlev; - - SDS.tke[offset] = tke_input[n]; - } - - for(Int n = 0; n < nlevi; ++n) { - const auto offset = n + s * nlevi; - - SDS.pint[offset] = pint[n]; - } - } - - // Check that the inputs make sense - - for(Int s = 0; s < shcol; ++s) { - for (Int n = 0; n < nlev; ++n){ - const auto offset = n + s * nlev; - - REQUIRE(SDS.tke[offset] >= mintke); - } - } - - // Call the fortran implementation - shoc_energy_threshold_fixer(SDS); - - // Verify the result - for(Int s = 0; s < shcol; ++s) { - // Make sure value of shoctop is within reasonable range - REQUIRE(SDS.shoctop[s] < nlev); - REQUIRE(SDS.shoctop[s] > 1); - - // Verify that shoctop represents what we want it to - // Make sure that thickness that bounds shoctop is positive - const auto offset_stopi = (SDS.shoctop[s]-1) + s * nlevi; - const auto offset_bot = (nlevi-1) + s * nlevi; - REQUIRE(SDS.pint[offset_bot] - SDS.pint[offset_stopi] > 0.0); - - if (SDS.shoctop[s] < nlev){ - const auto offset_stop = (SDS.shoctop[s]-1) + s * nlev; - REQUIRE(SDS.tke[offset_stop] == mintke); - REQUIRE(SDS.tke[offset_stop+1] > mintke); - } - } - - } - - static void run_bfb() - { - // TODO - } -}; - -} // namespace unit_test -} // namespace shoc -} // namespace scream - -namespace { - -TEST_CASE("shoc_energy_threshold_fixer_property", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocEnergyThreshFixer; - - TestStruct::run_property(); -} - -TEST_CASE("shoc_energy_threshold_fixer_bfb", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocEnergyThreshFixer; - - TestStruct::run_bfb(); -} - -} // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_energy_total_fixer_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_energy_total_fixer_tests.cpp deleted file mode 100644 index ea59b217f775..000000000000 --- a/components/eamxx/src/physics/shoc/tests/shoc_energy_total_fixer_tests.cpp +++ /dev/null @@ -1,167 +0,0 @@ -#include "catch2/catch.hpp" - -#include "shoc_unit_tests_common.hpp" - -#include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" -#include "physics/share/physics_constants.hpp" -#include "share/scream_types.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/util/ekat_arch.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" - -#include -#include -#include -#include - -namespace scream { -namespace shoc { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestShocTotEnergyFixer { - - static void run_property() - { - static constexpr Int shcol = 2; - static constexpr Int nlev = 5; - static constexpr auto nlevi = nlev + 1; - - // Tests for the SHOC function - // shoc_energy_total_fixer - - // FIRST TEST - // Surface flux test. Have two columns, one with zero surface fluxes - // and the other with positive surface fluxes. Verify the column - // with surface fluxes has greater total energy "before". - - // Timestep [s] - static constexpr Real dtime = 300; - // Number of macmic steps - static constexpr Int nadv = 2; - // Air density [km/m3] - static constexpr Real rho_zt[nlev] = {0.4, 0.6, 0.7, 0.9, 1.0}; - // Interface heights [m] - static constexpr Real zi_grid[nlevi] = {11000, 7500, 5000, 3000, 1500, 0}; - // Define integrated static energy, kinetic energy, water vapor, - // and liquid water respectively - static constexpr Real se = 200; - static constexpr Real ke = 150; - static constexpr Real wv = 0.5; - static constexpr Real wl = 0.1; - // Define surface sensible heat flux [K m/s] - static constexpr Real wthl_sfc = 0.5; - // Define surface total water flux [kg/kg m/s] - static constexpr Real wqw_sfc = 0.01; - // Pressure at interface [Pa] - static constexpr Real pint[nlevi] = {50000, 60000, 70000, 80000, 90000, 100000}; - - // Initialize data structure for bridging to F90 - ShocEnergyTotalFixerData SDS(shcol, nlev, nlevi, dtime, nadv); - - // Test that the inputs are reasonable. - // for this test we need exactly two columns - REQUIRE( (SDS.shcol == shcol && SDS.nlev == nlev && SDS.nlevi == nlevi && SDS.dtime == dtime && SDS.nadv == nadv) ); - REQUIRE(shcol == 2); - REQUIRE(nlevi == nlev+1); - - for(Int s = 0; s < shcol; ++s) { - // Set before and after integrals equal - SDS.se_a[s] = se; - SDS.se_b[s] = se; - SDS.ke_a[s] = ke; - SDS.ke_b[s] = ke; - SDS.wv_a[s] = wv; - SDS.wv_b[s] = wv; - SDS.wl_a[s] = wl; - SDS.wl_b[s] = wl; - - // Make first column be zero for the surface fluxes - SDS.wthl_sfc[s] = s*wthl_sfc; - SDS.wqw_sfc[s] = s*wqw_sfc; - - // Fill in test data on zt_grid. - for(Int n = 0; n < nlev; ++n) { - const auto offset = n + s * nlev; - - // For zt grid, set as midpoint of zi grid - SDS.zt_grid[offset] = 0.5*(zi_grid[n]+zi_grid[n+1]); - SDS.rho_zt[offset] = rho_zt[n]; - } - // Fill in test data on zi_grid. - for(Int n = 0; n < nlevi; ++n) { - const auto offset = n + s * nlevi; - - SDS.zi_grid[offset] = zi_grid[n]; - SDS.pint[offset] = pint[n]; - } - } - - // Check that the inputs make sense - - for(Int s = 0; s < shcol; ++s) { - for (Int n = 0; n < nlev; ++n){ - const auto offset = n + s * nlev; - - REQUIRE(SDS.zt_grid[offset] >= 0); - REQUIRE(SDS.rho_zt[offset] > 0); - - // Check that heights increase upward - if (n > nlev-1){ - REQUIRE(SDS.zt_grid[offset + 1] - SDS.zt_grid[offset] < 0); - } - } - for (Int n = 0; n < nlevi; ++n){ - const auto offset = n + s * nlevi; - - REQUIRE(SDS.zi_grid[offset] >= 0); - - // Check that heights increase upward - if (n > nlevi-1){ - REQUIRE(SDS.zi_grid[offset + 1] - SDS.zi_grid[offset] < 0); - } - } - } - - // Call the fortran implementation - shoc_energy_total_fixer(SDS); - - // Check test - - // For first column verify that total energies are the same - REQUIRE(SDS.te_a[0] == SDS.te_b[0]); - - // Verify that second column "before" energy is greater than - // the first column, since here we have active surface fluxes - REQUIRE(SDS.te_b[1] > SDS.te_b[0]); - } - - static void run_bfb() - { - // TODO - } -}; - -} // namespace unit_test -} // namespace shoc -} // namespace scream - -namespace { - -TEST_CASE("shoc_energy_total_fixer_property", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocTotEnergyFixer; - - TestStruct::run_property(); -} - -TEST_CASE("shoc_energy_total_fixer_bfb", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocTotEnergyFixer; - - TestStruct::run_bfb(); -} - -} // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_energy_update_dse_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_energy_update_dse_tests.cpp index a4e01eadb852..639d48fc70d7 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_energy_update_dse_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_energy_update_dse_tests.cpp @@ -3,7 +3,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -22,9 +22,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocUpdateDse { +struct UnitWrap::UnitTest::TestShocUpdateDse : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; static constexpr Int nlev = 5; @@ -109,11 +109,7 @@ struct UnitWrap::UnitTest::TestShocUpdateDse { } // Call the C++ implementation - SDS.transpose(); - // expects data in fortran layout - update_host_dse_f(SDS.shcol,SDS.nlev,SDS.thlm,SDS.shoc_ql,SDS.inv_exner,SDS.zt_grid, - SDS.phis,SDS.host_dse); - SDS.transpose(); + update_host_dse(SDS); // Check test for(Int s = 0; s < shcol; ++s) { @@ -140,11 +136,11 @@ struct UnitWrap::UnitTest::TestShocUpdateDse { } } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - UpdateHostDseData SDS_f90[] = { + UpdateHostDseData SDS_baseline[] = { // shcol, nlev UpdateHostDseData(10, 71), UpdateHostDseData(10, 12), @@ -153,46 +149,49 @@ struct UnitWrap::UnitTest::TestShocUpdateDse { }; // Generate random input data - for (auto& d : SDS_f90) { + for (auto& d : SDS_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state UpdateHostDseData SDS_cxx[] = { - UpdateHostDseData(SDS_f90[0]), - UpdateHostDseData(SDS_f90[1]), - UpdateHostDseData(SDS_f90[2]), - UpdateHostDseData(SDS_f90[3]), + UpdateHostDseData(SDS_baseline[0]), + UpdateHostDseData(SDS_baseline[1]), + UpdateHostDseData(SDS_baseline[2]), + UpdateHostDseData(SDS_baseline[3]), }; + static constexpr Int num_runs = sizeof(SDS_baseline) / sizeof(UpdateHostDseData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : SDS_f90) { - // expects data in C layout - update_host_dse(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : SDS_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : SDS_cxx) { - d.transpose(); - // expects data in fortran layout - update_host_dse_f(d.shcol,d.nlev,d.thlm,d.shoc_ql,d.inv_exner,d.zt_grid, - d.phis,d.host_dse); - d.transpose(); + update_host_dse(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(SDS_f90) / sizeof(UpdateHostDseData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - UpdateHostDseData& d_f90 = SDS_f90[i]; + UpdateHostDseData& d_baseline = SDS_baseline[i]; UpdateHostDseData& d_cxx = SDS_cxx[i]; - for (Int k = 0; k < d_f90.total(d_f90.host_dse); ++k) { - REQUIRE(d_f90.host_dse[k] == d_cxx.host_dse[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.host_dse); ++k) { + REQUIRE(d_baseline.host_dse[k] == d_cxx.host_dse[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + SDS_cxx[i].write(Base::m_fid); + } } } }; @@ -207,14 +206,14 @@ TEST_CASE("shoc_energy_host_dse_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocUpdateDse; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_energy_host_dse_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocUpdateDse; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_fterm_diag_third_moms_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_fterm_diag_third_moms_tests.cpp deleted file mode 100644 index 4fd800ca4adc..000000000000 --- a/components/eamxx/src/physics/shoc/tests/shoc_fterm_diag_third_moms_tests.cpp +++ /dev/null @@ -1,175 +0,0 @@ -#include "catch2/catch.hpp" - -#include "shoc_unit_tests_common.hpp" -#include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" -#include "physics/share/physics_constants.hpp" -#include "share/scream_types.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/util/ekat_arch.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" - -#include -#include -#include -#include - -namespace scream { -namespace shoc { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestFtermdiagThirdMoms { - - static void run_property() - { - - // Tests for the SHOC function: - // f0_to_f5_diag_third_shoc_moment - - // TEST ONE - // Zero test. Given no gradients, verify that relevant - // terms are zero. - - // 1/grid spacing [m-1] - constexpr static Real thedz = 0.1; - // 1/grid spacing for two grids [m-1] - constexpr static Real thedz2 = 0.05; - // bet2 term (ggr/thetal) - constexpr static Real bet2 = 0.0327; - // return to isotropy timescale [s] - constexpr static Real iso = 1000; - // liquid water flux [K m/s] - constexpr static Real wthl_sec_zero = 0.01; - // thetal variance [K^2] - constexpr static Real thl_sec_zero = 2; - // vertical velocity variance [m2/s2] - constexpr static Real w_sec_zero = 0.4; - // TKE [m2/s2] - constexpr static Real tke_zero = 0.5; - - // Initialize data structure for bridging to F90 - F0ToF5DiagThirdShocMomentData SDS; - - // Fill in data - SDS.thedz = thedz; - SDS.thedz2 = thedz2; - SDS.bet2 = bet2; - SDS.iso = iso; - SDS.isosqrd = iso*iso; - // for the following moments, feed each level - // the same value for this test - SDS.wthl_sec = wthl_sec_zero; - SDS.wthl_sec_kc = wthl_sec_zero; - SDS.wthl_sec_kb = wthl_sec_zero; - SDS.thl_sec_kc = thl_sec_zero; - SDS.thl_sec_kb = thl_sec_zero; - SDS.w_sec = w_sec_zero; - SDS.w_sec_kc = w_sec_zero; - SDS.w_sec_zi = w_sec_zero; - SDS.tke = tke_zero; - SDS.tke_kc = tke_zero; - - // Be sure inputs are as we expect - REQUIRE(SDS.thedz > 0); - REQUIRE(SDS.thedz2 > 0); - REQUIRE(SDS.wthl_sec_kc == SDS.wthl_sec_kb); - REQUIRE(SDS.thl_sec_kc == SDS.thl_sec_kb); - REQUIRE(SDS.w_sec_kc == SDS.w_sec); - REQUIRE(SDS.tke_kc == SDS.tke); - - // Call the fortran implementation - f0_to_f5_diag_third_shoc_moment(SDS); - - // Check result, make sure all outputs are zero - REQUIRE(SDS.f0 == 0); - REQUIRE(SDS.f1 == 0); - REQUIRE(SDS.f2 == 0); - REQUIRE(SDS.f3 == 0); - REQUIRE(SDS.f4 == 0); - REQUIRE(SDS.f5 == 0); - - // TEST TWO - // Positive gradient test. Feed the function values of the second - // moments with positive gradients. All fterms should have positive values - - // liquid water flux [K m/s] - constexpr static Real wthl_sec = 0.01; - // liquid water flux [K m/s] above - constexpr static Real wthl_sec_kc = 0.02; - // liquid water flux [K m/s] below - constexpr static Real wthl_sec_kb = 0; - // thetal variance [K^2] above - constexpr static Real thl_sec_kc = 2.5; - // thetal variance [K^2] - constexpr static Real thl_sec_kb = 1.7; - // vertical velocity variance [m2/s2] - constexpr static Real w_sec = 0.4; - // vertical velocity variance [m2/s2] above - constexpr static Real w_sec_kc = 0.5; - // TKE [m2/s2] - constexpr static Real tke = 0.5; - // TKE [m2/s2] above - constexpr static Real tke_kc = 0.55; - - // Feed in data - SDS.wthl_sec = wthl_sec; - SDS.wthl_sec_kc = wthl_sec_kc; - SDS.wthl_sec_kb = wthl_sec_kb; - SDS.thl_sec_kc = thl_sec_kc; - SDS.thl_sec_kb = thl_sec_kb; - SDS.w_sec = w_sec; - SDS.w_sec_kc = w_sec_kc; - SDS.w_sec_zi = w_sec; - SDS.tke = tke; - SDS.tke_kc = tke_kc; - - // Verify input is what we want for this test - REQUIRE(wthl_sec > 0); - REQUIRE(wthl_sec_kc > wthl_sec_kb); - REQUIRE(thl_sec_kc > thl_sec_kb); - REQUIRE(w_sec_kc > w_sec); - REQUIRE(tke_kc > tke); - - // Call the fortran implementation - f0_to_f5_diag_third_shoc_moment(SDS); - - // Check result, make sure all outputs are greater than zero - REQUIRE(SDS.f0 > 0); - REQUIRE(SDS.f1 > 0); - REQUIRE(SDS.f2 > 0); - REQUIRE(SDS.f3 > 0); - REQUIRE(SDS.f4 > 0); - REQUIRE(SDS.f5 > 0); - - } - - static void run_bfb() - { - // TODO - } - -}; - -} // namespace unit_test -} // namespace shoc -} // namespace scream - -namespace{ - -TEST_CASE("shoc_fterm_diag_third_moms_property", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestFtermdiagThirdMoms; - - TestStruct::run_property(); -} - -TEST_CASE("shoc_fterm_diag_third_moms_bfb", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestFtermdiagThirdMoms; - - TestStruct::run_bfb(); -} - -} // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_fterm_input_third_moms_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_fterm_input_third_moms_tests.cpp deleted file mode 100644 index 8d7774f1aba8..000000000000 --- a/components/eamxx/src/physics/shoc/tests/shoc_fterm_input_third_moms_tests.cpp +++ /dev/null @@ -1,112 +0,0 @@ -#include "catch2/catch.hpp" - -#include "shoc_unit_tests_common.hpp" -#include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" -#include "physics/share/physics_constants.hpp" -#include "share/scream_types.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/util/ekat_arch.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" - -#include -#include -#include -#include - -namespace scream { -namespace shoc { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestFtermInputThirdMoms { - - static void run_property() - { - - // Tests for the SHOC function: - // fterms_input_for_diag_third_shoc_moment - - // TEST - // Given inputs, verify that output is reasonable - - // grid spacing on interface grid [m] - constexpr static Real dz_zi = 100; - // grid spacing on midpoint grid [m] - constexpr static Real dz_zt = 80; - // grid spacing on adjacent midpoint grid [m] - constexpr static Real dz_zt_kc = 120; - // Return to isotropic timescale [s] - constexpr static Real isotropy_zi = 1000; - // Brunt vaisalla frequency [s] - constexpr static Real brunt_zi = -0.05; - // Potential temperature on interface grid [K] - constexpr static Real thetal_zi = 300; - - // Initialize data structure for bridging to F90 - FtermsInputForDiagThirdShocMomentData SDS; - - SDS.dz_zi = dz_zi; - SDS.dz_zt = dz_zt; - SDS.dz_zt_kc = dz_zt_kc; - SDS.isotropy_zi = isotropy_zi; - SDS.brunt_zi = brunt_zi; - SDS.thetal_zi = thetal_zi; - - // Check that input is physical - REQUIRE(SDS.dz_zi > 0); - REQUIRE(SDS.dz_zt > 0); - REQUIRE(SDS.dz_zt_kc > 0); - REQUIRE(SDS.isotropy_zi > 0); - REQUIRE(SDS.thetal_zi > 0); - - // Call the fortran implementation - fterms_input_for_diag_third_shoc_moment(SDS); - - // Verify the result - - // Check that thedz2 is smaller than thedz. - REQUIRE(SDS.thedz2 < SDS.thedz); - - // Check that bet2 is smaller than thetal - REQUIRE(SDS.bet2 < SDS.thetal_zi); - - // Be sure that iso and isosqrd relationships hold - if (SDS.isotropy_zi > 1){ - REQUIRE(SDS.isosqrd > SDS.iso); - } - else{ - REQUIRE(SDS.isosqrd < SDS.iso); - } - - } - - static void run_bfb() - { - // TODO - } - -}; - -} // namespace unit_test -} // namespace shoc -} // namespace scream - -namespace{ - -TEST_CASE("shoc_fterm_input_third_moms_property", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestFtermInputThirdMoms; - - TestStruct::run_property(); -} - -TEST_CASE("shoc_fterm_input_third_moms_bfb", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestFtermInputThirdMoms; - - TestStruct::run_bfb(); -} - -} // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_grid_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_grid_tests.cpp index 6a79eeca02ee..f9bd348dc47d 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_grid_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_grid_tests.cpp @@ -4,7 +4,7 @@ #include "physics/share/physics_constants.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -23,9 +23,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocGrid { +struct UnitWrap::UnitTest::TestShocGrid : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Real gravit = scream::physics::Constants::gravit; static constexpr Int shcol = 2; @@ -80,9 +80,7 @@ struct UnitWrap::UnitTest::TestShocGrid { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - shoc_grid_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.zt_grid, SDS.zi_grid, SDS.pdel, SDS.dz_zt, SDS.dz_zi, SDS.rho_zt); - SDS.transpose(); // go back to C layout + shoc_grid(SDS); // First check that dz is correct for(Int s = 0; s < shcol; ++s) { @@ -129,61 +127,66 @@ struct UnitWrap::UnitTest::TestShocGrid { } } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - ShocGridData f90_data[] = { + ShocGridData baseline_data[] = { ShocGridData(10, 71, 72), ShocGridData(10, 12, 13), ShocGridData(7, 16, 17), ShocGridData(2, 7, 8), }; + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(ShocGridData); + // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { + // Alternatively, you can use the baseline_data construtors/initializer lists to hardcode data + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state ShocGridData cxx_data[] = { - ShocGridData(f90_data[0]), - ShocGridData(f90_data[1]), - ShocGridData(f90_data[2]), - ShocGridData(f90_data[3]), + ShocGridData(baseline_data[0]), + ShocGridData(baseline_data[1]), + ShocGridData(baseline_data[2]), + ShocGridData(baseline_data[3]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - shoc_grid(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - shoc_grid_f(d.shcol, d.nlev, d.nlevi, d.zt_grid, d.zi_grid, d.pdel, d.dz_zt, d.dz_zi, d.rho_zt); - d.transpose(); // go back to C layout + shoc_grid(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(ShocGridData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - ShocGridData& d_f90 = f90_data[i]; + ShocGridData& d_baseline = baseline_data[i]; ShocGridData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.dz_zt); ++k) { - REQUIRE(d_f90.dz_zt[k] == d_cxx.dz_zt[k]); - REQUIRE(d_f90.rho_zt[k] == d_cxx.rho_zt[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.dz_zt); ++k) { + REQUIRE(d_baseline.dz_zt[k] == d_cxx.dz_zt[k]); + REQUIRE(d_baseline.rho_zt[k] == d_cxx.rho_zt[k]); } - for (Int k = 0; k < d_f90.total(d_f90.dz_zi); ++k) { - REQUIRE(d_f90.dz_zi[k] == d_cxx.dz_zi[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.dz_zi); ++k) { + REQUIRE(d_baseline.dz_zi[k] == d_cxx.dz_zi[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + cxx_data[i].write(Base::m_fid); + } } } // run_bfb }; @@ -198,14 +201,14 @@ TEST_CASE("shoc_grid_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocGrid; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_grid_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocGrid; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_impli_comp_tmpi_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_impli_comp_tmpi_tests.cpp index 53280dab17a6..1616bff130d8 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_impli_comp_tmpi_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_impli_comp_tmpi_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestImpCompTmpi { +struct UnitWrap::UnitTest::TestImpCompTmpi : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; static constexpr Int nlevi = 6; @@ -87,9 +87,7 @@ struct UnitWrap::UnitTest::TestImpCompTmpi { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - compute_tmpi_f(SDS.nlevi, SDS.shcol, SDS.dtime, SDS.rho_zi, SDS.dz_zi, SDS.tmpi); - SDS.transpose(); // go back to C layout + compute_tmpi(SDS); // Verify result for(Int s = 0; s < shcol; ++s) { @@ -118,11 +116,11 @@ struct UnitWrap::UnitTest::TestImpCompTmpi { } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - ComputeTmpiData f90_data[] = { + ComputeTmpiData baseline_data[] = { // shcol, nlevi, dtime ComputeTmpiData(10, 72, 1), ComputeTmpiData(10, 13, 10), @@ -131,44 +129,49 @@ struct UnitWrap::UnitTest::TestImpCompTmpi { }; // Generate random input data - for (auto& d : f90_data) { + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state ComputeTmpiData cxx_data[] = { - ComputeTmpiData(f90_data[0]), - ComputeTmpiData(f90_data[1]), - ComputeTmpiData(f90_data[2]), - ComputeTmpiData(f90_data[3]), + ComputeTmpiData(baseline_data[0]), + ComputeTmpiData(baseline_data[1]), + ComputeTmpiData(baseline_data[2]), + ComputeTmpiData(baseline_data[3]), }; + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(ComputeTmpiData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - compute_tmpi(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - compute_tmpi_f(d.nlevi, d.shcol, d.dtime, d.rho_zi, d.dz_zi, d.tmpi); - d.transpose(); // go back to C layout + compute_tmpi(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(ComputeTmpiData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - ComputeTmpiData& d_f90 = f90_data[i]; + ComputeTmpiData& d_baseline = baseline_data[i]; ComputeTmpiData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.tmpi); ++k) { - REQUIRE(d_f90.tmpi[k] == d_cxx.tmpi[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.tmpi); ++k) { + REQUIRE(d_baseline.tmpi[k] == d_cxx.tmpi[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + cxx_data[i].write(Base::m_fid); + } } } }; @@ -183,14 +186,14 @@ TEST_CASE("shoc_imp_comp_tmpi_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestImpCompTmpi; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_imp_comp_tmpi_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestImpCompTmpi; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_impli_dp_inverse_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_impli_dp_inverse_tests.cpp index aad9c0266869..b5feb285f559 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_impli_dp_inverse_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_impli_dp_inverse_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestImpDpInverse { +struct UnitWrap::UnitTest::TestImpDpInverse : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; static constexpr Int nlev = 5; @@ -74,9 +74,7 @@ struct UnitWrap::UnitTest::TestImpDpInverse { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - dp_inverse_f(SDS.nlev, SDS.shcol, SDS.rho_zt, SDS.dz_zt, SDS.rdp_zt); - SDS.transpose(); // go back to C layout + dp_inverse(SDS); // Verify result for(Int s = 0; s < shcol; ++s) { @@ -98,11 +96,11 @@ struct UnitWrap::UnitTest::TestImpDpInverse { } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - DpInverseData f90_data[] = { + DpInverseData baseline_data[] = { // shcol, nlev DpInverseData(10, 71), DpInverseData(10, 12), @@ -111,44 +109,49 @@ struct UnitWrap::UnitTest::TestImpDpInverse { }; // Generate random input data - for (auto& d : f90_data) { + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state DpInverseData cxx_data[] = { - DpInverseData(f90_data[0]), - DpInverseData(f90_data[1]), - DpInverseData(f90_data[2]), - DpInverseData(f90_data[3]), + DpInverseData(baseline_data[0]), + DpInverseData(baseline_data[1]), + DpInverseData(baseline_data[2]), + DpInverseData(baseline_data[3]), }; + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(DpInverseData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - dp_inverse(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - dp_inverse_f(d.nlev, d.shcol, d.rho_zt, d.dz_zt, d.rdp_zt); - d.transpose(); // go back to C layout + dp_inverse(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(DpInverseData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - DpInverseData& d_f90 = f90_data[i]; + DpInverseData& d_baseline = baseline_data[i]; DpInverseData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.rdp_zt); ++k) { - REQUIRE(d_f90.rdp_zt[k] == d_cxx.rdp_zt[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.rdp_zt); ++k) { + REQUIRE(d_baseline.rdp_zt[k] == d_cxx.rdp_zt[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + cxx_data[i].write(Base::m_fid); + } } } }; @@ -163,14 +166,14 @@ TEST_CASE("shoc_imp_dp_inverse_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestImpDpInverse; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_imp_dp_inverse_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestImpDpInverse; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_impli_sfc_fluxes_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_impli_sfc_fluxes_tests.cpp deleted file mode 100644 index 8f8f25cdd4ac..000000000000 --- a/components/eamxx/src/physics/shoc/tests/shoc_impli_sfc_fluxes_tests.cpp +++ /dev/null @@ -1,202 +0,0 @@ -#include "catch2/catch.hpp" - -#include "shoc_unit_tests_common.hpp" -#include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" -#include "physics/share/physics_constants.hpp" -#include "share/scream_types.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/util/ekat_arch.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" - -#include -#include -#include -#include - -namespace scream { -namespace shoc { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestImpSfcFluxes { - - static void run_property() - { - static constexpr Int shcol = 5; - static constexpr Int num_tracer = 10; - - // Tests for the SHOC subroutine - // sfc_fluxes - - // TEST - // Feed in several columns worth of data and make sure - // the output is consistent. - - // Surface density on the zi grid [kg/m3] - static constexpr Real rho_zi_sfc[shcol] = {1.2, 1.0, 0.9, 1.1, 1.15}; - // Rdp value on zt grid [ms^2/kg], same for all columns - static constexpr Real rdp_zt_sfc = 8.5e-3; - // heat flux at surface [K m/s] - static constexpr Real wthl_sfc[shcol] = {0.03, -0.03, 0.1, 0, -0.1}; - // moisture flux at surface [kg/kg m/s] - static constexpr Real wqw_sfc[shcol] = {2e-5, 1e-6, 0, -2e-5, 1e-4}; - // TKE flux at the surface [m3/s3] - static constexpr Real wtke_sfc[shcol] = {4e-2, 1e-3, -2e-3, 0, -1e-3}; - - // Supply input values - // liquid water potential temperature [K] - static constexpr Real thetal_in = 300; - // total water mixing ratio [kg/kg] - static constexpr Real qw_in = 0.015; - // turbulent kinetic energy [m2/s2] - static constexpr Real tke_in = 0.4; - - // time step [s] - static constexpr Real dtime = 300; - - // Input for tracer (no units) - Real tracer_in[num_tracer]; - - // Feed tracer random data from 1 to 1000 - for(Int t = 0; t < num_tracer; ++t) { - tracer_in[t] = rand()% 1000 + 1; - } - - // Initialize data structure for bridging to F90 - SfcFluxesData SDS(shcol, num_tracer, dtime); - - // Test that the inputs are reasonable. - REQUIRE(SDS.shcol == shcol); - REQUIRE(SDS.num_tracer == num_tracer); - REQUIRE(shcol > 1); - - // Fill in test data, column only - for(Int s = 0; s < shcol; ++s) { - SDS.rho_zi_sfc[s] = rho_zi_sfc[s]; - SDS.rdp_zt_sfc[s] = rdp_zt_sfc; - SDS.wthl_sfc[s] = wthl_sfc[s]; - SDS.wqw_sfc[s] = wqw_sfc[s]; - SDS.wtke_sfc[s] = wtke_sfc[s]; - - SDS.thetal[s] = thetal_in; - SDS.qw[s] = qw_in; - SDS.tke[s] = tke_in; - - for (Int t = 0; t < num_tracer; ++t){ - const auto offset = t + s * num_tracer; - SDS.wtracer[offset] = tracer_in[t]; - // Feed tracer flux random data from -100 to 100 - // note this is different for every point - SDS.wtracer_sfc[offset] = rand()% 200 + (-100); - } - - } - - // Check that the inputs make sense - for(Int s = 0; s < shcol; ++s) { - REQUIRE( (SDS.thetal[s] > 150 && SDS.thetal[s] < 350) ); - REQUIRE( (SDS.qw[s] > 0.0001 && SDS.qw[s] < 0.05) ); - REQUIRE( (SDS.tke[s] > 0 && SDS.tke[s] < 10) ); - REQUIRE( (SDS.rdp_zt_sfc[s] > 0 && SDS.rdp_zt_sfc[s] < 1) ); - REQUIRE( (SDS.rho_zi_sfc[s] > 0 && SDS.rho_zi_sfc[s] < 2) ); - REQUIRE(std::abs(SDS.wthl_sfc[s]) < 1); - REQUIRE(std::abs(SDS.wqw_sfc[s]) < 1e-3); - REQUIRE(std::abs(SDS.wtke_sfc[s]) < 0.1); - REQUIRE(SDS.dtime > 0); - } - - // Call the fortran implementation - sfc_fluxes(SDS); - - // Verify that output is reasonable - for(Int s = 0; s < shcol; ++s) { - - // Verify output falls within reasonable bounds - REQUIRE( (SDS.thetal[s] > 150 && SDS.thetal[s] < 350) ); - REQUIRE( (SDS.qw[s] > 0.0001 && SDS.qw[s] < 0.05) ); - REQUIRE( (SDS.tke[s] > 0 && SDS.tke[s] < 10) ); - - // Based on surface flux input, make sure that - // temperature, moisture, and tke all have output - // that is expected with respect to the input - - // Check temperature - if (wthl_sfc[s] > 0){ - REQUIRE(SDS.thetal[s] > thetal_in); - } - else if (wthl_sfc[s] < 0){ - REQUIRE(SDS.thetal[s] < thetal_in); - } - else{ - REQUIRE(SDS.thetal[s] == thetal_in); - } - - // Check moisture - if (wqw_sfc[s] > 0){ - REQUIRE(SDS.qw[s] > qw_in); - } - else if (wqw_sfc[s] < 0){ - REQUIRE(SDS.qw[s] < qw_in); - } - else{ - REQUIRE(SDS.qw[s] == qw_in); - } - - // Check TKE - if (wtke_sfc[s] > 0){ - REQUIRE(SDS.tke[s] > tke_in); - } - else if (wtke_sfc[s] < 0){ - REQUIRE(SDS.tke[s] < tke_in); - } - else{ - REQUIRE(SDS.tke[s] == tke_in); - } - - // Check tracer - for (Int t = 0; t < num_tracer; ++t){ - const auto offset = t + s * num_tracer; - if (SDS.wtracer_sfc[offset] > 0){ - REQUIRE(SDS.wtracer[offset] > tracer_in[t]); - } - else if (SDS.wtracer_sfc[offset] < 0){ - REQUIRE(SDS.wtracer[offset] < tracer_in[t]); - } - else{ - REQUIRE(SDS.wtracer[offset] == tracer_in[t]); - } - } - - } - - } - - static void run_bfb() - { - // TODO - } -}; - -} // namespace unit_test -} // namespace shoc -} // namespace scream - -namespace { - -TEST_CASE("shoc_imp_sfc_fluxes_property", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestImpSfcFluxes; - - TestStruct::run_property(); -} - -TEST_CASE("shoc_imp_sfc_fluxes_bfb", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestImpSfcFluxes; - - TestStruct::run_bfb(); -} - -} // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_impli_srf_stress_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_impli_srf_stress_tests.cpp deleted file mode 100644 index 97f6d2a0be3b..000000000000 --- a/components/eamxx/src/physics/shoc/tests/shoc_impli_srf_stress_tests.cpp +++ /dev/null @@ -1,131 +0,0 @@ -#include "catch2/catch.hpp" - -#include "shoc_unit_tests_common.hpp" -#include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" -#include "physics/share/physics_constants.hpp" -#include "share/scream_types.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/util/ekat_arch.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" - -#include -#include -#include -#include - -namespace scream { -namespace shoc { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestImpSfcStress { - - static void run_property() - { - static constexpr Int shcol = 5; - - // Tests for the SHOC subroutine - // impli_srf_stress_term - - // TEST ONE - // Feed in several columns worth of data and make sure - // the output is consistent. - - // Surface density on the zi grid [kg/m3] - static constexpr Real rho_zi_sfc[shcol] = {1.2, 1.0, 0.9, 1.1, 1.15}; - // Surface moment flux, zonal direction [m3/s3] - static constexpr Real uw_sfc[shcol] = {0.03, -0.03, 0.1, 0, -0.1}; - // Surface moment flux, meridional direction [m3/s3] - static constexpr Real vw_sfc[shcol] = {-0.01, -0.01, 0.3, 0, -0.3}; - // Surface wind, zonal direction [m/s] - static constexpr Real u_wind_sfc[shcol] = {5, -5, 0, 2, -10}; - // Surface wind, meridional direction [m/s] - static constexpr Real v_wind_sfc[shcol] = {-10, 2, 20, 0, 1}; - - // Initialize data structure for bridging to F90 - ImpliSrfStressTermData SDS(shcol); - - // Test that the inputs are reasonable. - REQUIRE(SDS.shcol == shcol); - REQUIRE(shcol > 1); - - // Fill in test data, column only - for(Int s = 0; s < shcol; ++s) { - SDS.rho_zi_sfc[s] = rho_zi_sfc[s]; - SDS.uw_sfc[s] = uw_sfc[s]; - SDS.vw_sfc[s] = vw_sfc[s]; - SDS.u_wind_sfc[s] = u_wind_sfc[s]; - SDS.v_wind_sfc[s] = v_wind_sfc[s]; - } - - // Call the fortran implementation - impli_srf_stress_term(SDS); - - // Verify that output is reasonable - for(Int s = 0; s < shcol; ++s) { - // term should be greater than zero and less than one given - // reasonable input values - REQUIRE( (SDS.ksrf[s] > 0 && SDS.ksrf[s] < 1) ); - } - - // TEST TWO - // Given inputs that are identical but the absolute value of - // the surface fluxes are INCREASING, verify ksrf value is larger. - // Can recycle input from surface fluxes from last test. - - // Fill in test data, column only - for(Int s = 0; s < shcol; ++s) { - SDS.rho_zi_sfc[s] = 1.2; // density [kg/m3] - SDS.u_wind_sfc[s] = 5; // zonal wind [m/s] - SDS.v_wind_sfc[s] = -10; // meridional wind [m/s] - } - - // Call the fortran implementation - impli_srf_stress_term(SDS); - - Real stress1, stress2; - // Verify that output is as expected and reasonable - for(Int s = 0; s < shcol; ++s) { - // term should be greater than zero and less than one given - // reasonable input values - REQUIRE( (SDS.ksrf[s] > 0 && SDS.ksrf[s] < 1) ); - if (s < shcol-1){ - stress1 = uw_sfc[s]*uw_sfc[s] + uw_sfc[s]*uw_sfc[s]; - stress2 = uw_sfc[s+1]*uw_sfc[s+1] + uw_sfc[s+1]*uw_sfc[s+1]; - if (stress1 > stress2){ - REQUIRE(SDS.ksrf[s] > SDS.ksrf[s+1]); - } - } - } - - } - - static void run_bfb() - { - // TODO - } -}; - -} // namespace unit_test -} // namespace shoc -} // namespace scream - -namespace { - -TEST_CASE("shoc_imp_sfc_stress_property", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestImpSfcStress; - - TestStruct::run_property(); -} - -TEST_CASE("shoc_imp_sfc_stress_bfb", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestImpSfcStress; - - TestStruct::run_bfb(); -} - -} // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_impli_srf_tke_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_impli_srf_tke_tests.cpp deleted file mode 100644 index bb19e21e30c5..000000000000 --- a/components/eamxx/src/physics/shoc/tests/shoc_impli_srf_tke_tests.cpp +++ /dev/null @@ -1,101 +0,0 @@ -#include "catch2/catch.hpp" - -#include "shoc_unit_tests_common.hpp" -#include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" -#include "physics/share/physics_constants.hpp" -#include "share/scream_types.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/util/ekat_arch.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" - -#include -#include -#include -#include - -namespace scream { -namespace shoc { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestImpTkeSfcStress { - - static void run_property() - { - static constexpr Int shcol = 5; - - // Tests for the SHOC subroutine - // impli_srf_stress_term - - // TEST ONE - // Feed in several columns worth of data and make sure - // the output is consistent. Make sure that columns with higher - // surface stress result in greater surface tke flux. - - // Surface moment flux, zonal direction [m3/s3] - static constexpr Real uw_sfc[shcol] = {0.03, -0.03, 0.1, 0, -0.1}; - // Surface moment flux, meridional direction [m3/s3] - static constexpr Real vw_sfc[shcol] = {-0.01, -0.01, 0.3, 0, -0.3}; - - // Initialize data structure for bridging to F90 - TkeSrfFluxTermData SDS(shcol); - - // Test that the inputs are reasonable. - REQUIRE(SDS.shcol == shcol); - REQUIRE(shcol > 1); - - // Fill in test data, column only - for(Int s = 0; s < shcol; ++s) { - SDS.uw_sfc[s] = uw_sfc[s]; - SDS.vw_sfc[s] = vw_sfc[s]; - } - - // Call the fortran implementation - tke_srf_flux_term(SDS); - - Real stress1, stress2; - // Verify that output is as expected and reasonable - for(Int s = 0; s < shcol; ++s) { - // term should be greater than zero and less than one given - // reasonable input values - REQUIRE(SDS.wtke_sfc[s] > 0); - if (s < shcol-1){ - stress1 = uw_sfc[s]*uw_sfc[s] + uw_sfc[s]*uw_sfc[s]; - stress2 = uw_sfc[s+1]*uw_sfc[s+1] + uw_sfc[s+1]*uw_sfc[s+1]; - if (stress1 > stress2){ - REQUIRE(SDS.wtke_sfc[s] > SDS.wtke_sfc[s+1]); - } - } - } - - } - - static void run_bfb() - { - // TODO - } -}; - -} // namespace unit_test -} // namespace shoc -} // namespace scream - -namespace { - -TEST_CASE("shoc_imp_tkesfc_stress_property", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestImpTkeSfcStress; - - TestStruct::run_property(); -} - -TEST_CASE("shoc_imp_tkesfc_stress_bfb", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestImpTkeSfcStress; - - TestStruct::run_bfb(); -} - -} // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_l_inf_length_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_l_inf_length_tests.cpp index 84faaebe4b91..db769d170107 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_l_inf_length_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_l_inf_length_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestLInfShocLength { +struct UnitWrap::UnitTest::TestLInfShocLength : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 3; static constexpr Int nlev = 5; @@ -89,10 +89,7 @@ struct UnitWrap::UnitTest::TestLInfShocLength { } // Call the C++ implementation - SDS.transpose(); - // expects data in fortran layout - compute_l_inf_shoc_length_f(SDS.nlev,SDS.shcol,SDS.zt_grid,SDS.dz_zt,SDS.tke,SDS.l_inf); - SDS.transpose(); + compute_l_inf_shoc_length(SDS); // Check the results // Make sure result is bounded correctly @@ -108,11 +105,11 @@ struct UnitWrap::UnitTest::TestLInfShocLength { } } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - ComputeLInfShocLengthData SDS_f90[] = { + ComputeLInfShocLengthData SDS_baseline[] = { // shcol, nlev ComputeLInfShocLengthData(10, 71), ComputeLInfShocLengthData(10, 12), @@ -121,45 +118,49 @@ struct UnitWrap::UnitTest::TestLInfShocLength { }; // Generate random input data - for (auto& d : SDS_f90) { + for (auto& d : SDS_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state ComputeLInfShocLengthData SDS_cxx[] = { - ComputeLInfShocLengthData(SDS_f90[0]), - ComputeLInfShocLengthData(SDS_f90[1]), - ComputeLInfShocLengthData(SDS_f90[2]), - ComputeLInfShocLengthData(SDS_f90[3]), + ComputeLInfShocLengthData(SDS_baseline[0]), + ComputeLInfShocLengthData(SDS_baseline[1]), + ComputeLInfShocLengthData(SDS_baseline[2]), + ComputeLInfShocLengthData(SDS_baseline[3]), }; + static constexpr Int num_runs = sizeof(SDS_baseline) / sizeof(ComputeLInfShocLengthData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : SDS_f90) { - // expects data in C layout - compute_l_inf_shoc_length(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : SDS_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : SDS_cxx) { - d.transpose(); - // expects data in fortran layout - compute_l_inf_shoc_length_f(d.nlev,d.shcol,d.zt_grid,d.dz_zt,d.tke,d.l_inf); - d.transpose(); + compute_l_inf_shoc_length(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(SDS_f90) / sizeof(ComputeLInfShocLengthData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - ComputeLInfShocLengthData& d_f90 = SDS_f90[i]; + ComputeLInfShocLengthData& d_baseline = SDS_baseline[i]; ComputeLInfShocLengthData& d_cxx = SDS_cxx[i]; - for (Int c = 0; c < d_f90.shcol; ++c) { - REQUIRE(d_f90.l_inf[c] == d_cxx.l_inf[c]); + for (Int c = 0; c < d_baseline.shcol; ++c) { + REQUIRE(d_baseline.l_inf[c] == d_cxx.l_inf[c]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + SDS_cxx[i].write(Base::m_fid); + } } } }; @@ -174,14 +175,14 @@ TEST_CASE("shoc_l_inf_length_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestLInfShocLength; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_l_inf_length_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestLInfShocLength; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_length_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_length_tests.cpp index 0f890db9ce11..b492f6a8e4bf 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_length_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_length_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocLength { +struct UnitWrap::UnitTest::TestShocLength : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Real minlen = scream::shoc::Constants::minlen; static constexpr Real maxlen = scream::shoc::Constants::maxlen; @@ -121,12 +121,7 @@ struct UnitWrap::UnitTest::TestShocLength { } // Call the C++ implementation - SDS.transpose(); - // expects data in fortran layout - shoc_length_f(SDS.shcol,SDS.nlev,SDS.nlevi,SDS.host_dx,SDS.host_dy, - SDS.zt_grid,SDS.zi_grid,SDS.dz_zt,SDS.tke, - SDS.thv,SDS.brunt,SDS.shoc_mix); - SDS.transpose(); + shoc_length(SDS); // Verify output for(Int s = 0; s < shcol; ++s) { @@ -175,12 +170,7 @@ struct UnitWrap::UnitTest::TestShocLength { } // call C++ implentation - SDS.transpose(); - // expects data in fortran layout - shoc_length_f(SDS.shcol,SDS.nlev,SDS.nlevi,SDS.host_dx,SDS.host_dy, - SDS.zt_grid,SDS.zi_grid,SDS.dz_zt,SDS.tke, - SDS.thv,SDS.brunt,SDS.shoc_mix); - SDS.transpose(); + shoc_length(SDS); // Verify output for(Int s = 0; s < shcol; ++s) { @@ -196,11 +186,11 @@ struct UnitWrap::UnitTest::TestShocLength { } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - ShocLengthData SDS_f90[] = { + ShocLengthData SDS_baseline[] = { // shcol, nlev, nlevi ShocLengthData(12, 71, 72), ShocLengthData(10, 12, 13), @@ -209,48 +199,50 @@ struct UnitWrap::UnitTest::TestShocLength { }; // Generate random input data - for (auto& d : SDS_f90) { + for (auto& d : SDS_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state ShocLengthData SDS_cxx[] = { - ShocLengthData(SDS_f90[0]), - ShocLengthData(SDS_f90[1]), - ShocLengthData(SDS_f90[2]), - ShocLengthData(SDS_f90[3]), + ShocLengthData(SDS_baseline[0]), + ShocLengthData(SDS_baseline[1]), + ShocLengthData(SDS_baseline[2]), + ShocLengthData(SDS_baseline[3]), }; + static constexpr Int num_runs = sizeof(SDS_baseline) / sizeof(ShocLengthData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : SDS_f90) { - // expects data in C layout - shoc_length(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : SDS_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : SDS_cxx) { - d.transpose(); - // expects data in fortran layout - shoc_length_f(d.shcol,d.nlev,d.nlevi,d.host_dx,d.host_dy, - d.zt_grid,d.zi_grid,d.dz_zt,d.tke, - d.thv,d.brunt,d.shoc_mix); - d.transpose(); + shoc_length(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(SDS_f90) / sizeof(ShocLengthData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - ShocLengthData& d_f90 = SDS_f90[i]; + ShocLengthData& d_baseline = SDS_baseline[i]; ShocLengthData& d_cxx = SDS_cxx[i]; - for (Int k = 0; k < d_f90.total(d_f90.brunt); ++k) { - REQUIRE(d_f90.brunt[k] == d_cxx.brunt[k]); - REQUIRE(d_f90.shoc_mix[k] == d_cxx.shoc_mix[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.brunt); ++k) { + REQUIRE(d_baseline.brunt[k] == d_cxx.brunt[k]); + REQUIRE(d_baseline.shoc_mix[k] == d_cxx.shoc_mix[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + SDS_cxx[i].write(Base::m_fid); + } } } }; @@ -265,14 +257,14 @@ TEST_CASE("shoc_length_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocLength; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_length_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocLength; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_linear_interp_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_linear_interp_tests.cpp index 0a9c666688b5..6be406bda093 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_linear_interp_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_linear_interp_tests.cpp @@ -3,7 +3,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -22,9 +22,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocLinearInt { +struct UnitWrap::UnitTest::TestShocLinearInt : public UnitWrap::UnitTest::Base { - static void run_property_fixed() + void run_property_fixed() { static constexpr Int shcol = 2; static constexpr Int km1 = 5; @@ -100,9 +100,7 @@ struct UnitWrap::UnitTest::TestShocLinearInt { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - linear_interp_f(SDS.x1, SDS.x2, SDS.y1, SDS.y2, SDS.km1, SDS.km2, SDS.ncol, SDS.minthresh); - SDS.transpose(); // go back to C layout + linear_interp(SDS); // First check that all output temperatures are greater than zero @@ -194,9 +192,7 @@ struct UnitWrap::UnitTest::TestShocLinearInt { } // Call the C++ implementation - SDS2.transpose(); // _f expects data in fortran layout - linear_interp_f(SDS2.x1, SDS2.x2, SDS2.y1, SDS2.y2, SDS2.km1, SDS2.km2, SDS2.ncol, SDS2.minthresh); - SDS2.transpose(); // go back to C layout + linear_interp(SDS2); // Check the result, make sure output is bounded correctly @@ -223,7 +219,7 @@ struct UnitWrap::UnitTest::TestShocLinearInt { } } - static void run_property_random(bool km1_bigger) + void run_property_random(bool km1_bigger) { std::default_random_engine generator; std::pair km1_range = {13, 25}; @@ -281,9 +277,7 @@ struct UnitWrap::UnitTest::TestShocLinearInt { } // Call the C++ implementation - d.transpose(); // _f expects data in fortran layout - linear_interp_f(d.x1, d.x2, d.y1, d.y2, d.km1, d.km2, d.ncol, d.minthresh); - d.transpose(); // go back to C layout + linear_interp(d); // The combination of single-precision and randomness generating points // close together can result in larger error margins. @@ -333,18 +327,18 @@ struct UnitWrap::UnitTest::TestShocLinearInt { } } - static void run_property() + void run_property() { run_property_fixed(); run_property_random(true); run_property_random(false); } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - LinearInterpData f90_data[] = { + LinearInterpData baseline_data[] = { // shcol, nlev(km1), nlevi(km2), minthresh LinearInterpData(10, 72, 71, 1e-15), LinearInterpData(10, 71, 72, 1e-15), @@ -355,46 +349,50 @@ struct UnitWrap::UnitTest::TestShocLinearInt { }; // Generate random input data - for (auto& d : f90_data) { + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state LinearInterpData cxx_data[] = { - LinearInterpData(f90_data[0]), - LinearInterpData(f90_data[1]), - LinearInterpData(f90_data[2]), - LinearInterpData(f90_data[3]), - LinearInterpData(f90_data[4]), - LinearInterpData(f90_data[5]), + LinearInterpData(baseline_data[0]), + LinearInterpData(baseline_data[1]), + LinearInterpData(baseline_data[2]), + LinearInterpData(baseline_data[3]), + LinearInterpData(baseline_data[4]), + LinearInterpData(baseline_data[5]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - linear_interp(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - linear_interp_f(d.x1, d.x2, d.y1, d.y2, d.km1, d.km2, d.ncol, d.minthresh); - d.transpose(); // go back to C layout + linear_interp(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(LinearInterpData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(LinearInterpData); for (Int i = 0; i < num_runs; ++i) { - LinearInterpData& d_f90 = f90_data[i]; + LinearInterpData& d_baseline = baseline_data[i]; LinearInterpData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.y2); ++k) { - REQUIRE(d_f90.y2[k] == d_cxx.y2[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.y2); ++k) { + REQUIRE(d_baseline.y2[k] == d_cxx.y2[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } } // run_bfb @@ -410,14 +408,14 @@ TEST_CASE("shoc_linear_interp_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocLinearInt; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_linear_interp_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocLinearInt; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_main_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_main_tests.cpp index 5c3eb8f9860e..b007da41322e 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_main_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_main_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "shoc_unit_tests_common.hpp" @@ -14,9 +14,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocMain { +struct UnitWrap::UnitTest::TestShocMain : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Real mintke = scream::shoc::Constants::mintke; static constexpr Real minlen = scream::shoc::Constants::minlen; @@ -259,18 +259,7 @@ struct UnitWrap::UnitTest::TestShocMain { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - const int npbl = shoc_init_f(SDS.nlev, SDS.pref_mid, SDS.nbot_shoc, SDS.ntop_shoc); - - shoc_main_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.dtime, SDS.nadv, npbl, SDS.host_dx, SDS.host_dy, - SDS.thv, SDS.zt_grid, SDS.zi_grid, SDS.pres, SDS.presi, SDS.pdel, SDS.wthl_sfc, - SDS.wqw_sfc, SDS.uw_sfc, SDS.vw_sfc, SDS.wtracer_sfc, SDS.num_qtracers, - SDS.w_field, SDS.inv_exner, SDS.phis, SDS.host_dse, SDS.tke, SDS.thetal, SDS.qw, - SDS.u_wind, SDS.v_wind, SDS.qtracers, SDS.wthv_sec, SDS.tkh, SDS.tk, SDS.shoc_ql, - SDS.shoc_cldfrac, SDS.pblh, SDS.shoc_mix, SDS.isotropy, SDS.w_sec, SDS.thl_sec, - SDS.qw_sec, SDS.qwthl_sec, SDS.wthl_sec, SDS.wqw_sec, SDS.wtke_sec, SDS.uw_sec, - SDS.vw_sec, SDS.w3, SDS.wqls_sec, SDS.brunt, SDS.shoc_ql2); - SDS.transpose(); // go back to C layout + shoc_main(SDS); // Make sure output falls within reasonable bounds for(Int s = 0; s < shcol; ++s) { @@ -339,11 +328,11 @@ struct UnitWrap::UnitTest::TestShocMain { } // run_property - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - ShocMainData f90_data[] = { + ShocMainData baseline_data[] = { // shcol, nlev, nlevi, num_qtracers, dtime, nadv, nbot_shoc, ntop_shoc(C++ indexing) ShocMainData(12, 72, 73, 5, 300, 15, 72, 0), ShocMainData(8, 12, 13, 3, 300, 10, 8, 3), @@ -352,7 +341,7 @@ struct UnitWrap::UnitTest::TestShocMain { }; // Generate random input data - for (auto& d : f90_data) { + for (auto& d : baseline_data) { d.randomize(engine, { {d.presi, {700e2,1000e2}}, @@ -374,113 +363,108 @@ struct UnitWrap::UnitTest::TestShocMain { }); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state ShocMainData cxx_data[] = { - ShocMainData(f90_data[0]), - ShocMainData(f90_data[1]), - ShocMainData(f90_data[2]), - ShocMainData(f90_data[3]) + ShocMainData(baseline_data[0]), + ShocMainData(baseline_data[1]), + ShocMainData(baseline_data[2]), + ShocMainData(baseline_data[3]) }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - shoc_main_with_init(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - const int npbl = shoc_init_f(d.nlev, d.pref_mid, d.nbot_shoc, d.ntop_shoc); - - shoc_main_f(d.shcol, d.nlev, d.nlevi, d.dtime, d.nadv, npbl, d.host_dx, d.host_dy, - d.thv, d.zt_grid, d.zi_grid, d.pres, d.presi, d.pdel, d.wthl_sfc, - d.wqw_sfc, d.uw_sfc, d.vw_sfc, d.wtracer_sfc, d.num_qtracers, - d.w_field, d.inv_exner, d.phis, d.host_dse, d.tke, d.thetal, d.qw, - d.u_wind, d.v_wind, d.qtracers, d.wthv_sec, d.tkh, d.tk, d.shoc_ql, - d.shoc_cldfrac, d.pblh, d.shoc_mix, d.isotropy, d.w_sec, d.thl_sec, - d.qw_sec, d.qwthl_sec, d.wthl_sec, d.wqw_sec, d.wtke_sec, d.uw_sec, - d.vw_sec, d.w3, d.wqls_sec, d.brunt, d.shoc_ql2); - d.transpose(); // go back to C layout + shoc_main(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(ShocMainData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(ShocMainData); for (Int i = 0; i < num_runs; ++i) { - ShocMainData& d_f90 = f90_data[i]; + ShocMainData& d_baseline = baseline_data[i]; ShocMainData& d_cxx = cxx_data[i]; - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.host_dse)); - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.tke)); - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.thetal)); - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.qw)); - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.u_wind)); - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.v_wind)); - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.wthv_sec)); - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.tkh)); - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.tk)); - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.shoc_ql)); - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.shoc_cldfrac)); - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.shoc_mix)); - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.isotropy)); - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.w_sec)); - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.wqls_sec)); - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.brunt)); - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.shoc_ql2)); - for (Int k = 0; k < d_f90.total(d_f90.host_dse); ++k) { - REQUIRE(d_f90.host_dse[k] == d_cxx.host_dse[k]); - REQUIRE(d_f90.tke[k] == d_cxx.tke[k]); - REQUIRE(d_f90.thetal[k] == d_cxx.thetal[k]); - REQUIRE(d_f90.qw[k] == d_cxx.qw[k]); - REQUIRE(d_f90.u_wind[k] == d_cxx.u_wind[k]); - REQUIRE(d_f90.v_wind[k] == d_cxx.v_wind[k]); - REQUIRE(d_f90.wthv_sec[k] == d_cxx.wthv_sec[k]); - REQUIRE(d_f90.tk[k] == d_cxx.tk[k]); - REQUIRE(d_f90.shoc_ql[k] == d_cxx.shoc_ql[k]); - REQUIRE(d_f90.shoc_cldfrac[k] == d_cxx.shoc_cldfrac[k]); - REQUIRE(d_f90.shoc_mix[k] == d_cxx.shoc_mix[k]); - REQUIRE(d_f90.isotropy[k] == d_cxx.isotropy[k]); - REQUIRE(d_f90.w_sec[k] == d_cxx.w_sec[k]); - REQUIRE(d_f90.wqls_sec[k] == d_cxx.wqls_sec[k]); - REQUIRE(d_f90.brunt[k] == d_cxx.brunt[k]); - REQUIRE(d_f90.shoc_ql2[k] == d_cxx.shoc_ql2[k]); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.host_dse)); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.tke)); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.thetal)); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.qw)); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.u_wind)); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.v_wind)); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.wthv_sec)); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.tkh)); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.tk)); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.shoc_ql)); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.shoc_cldfrac)); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.shoc_mix)); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.isotropy)); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.w_sec)); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.wqls_sec)); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.brunt)); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.shoc_ql2)); + for (Int k = 0; k < d_baseline.total(d_baseline.host_dse); ++k) { + REQUIRE(d_baseline.host_dse[k] == d_cxx.host_dse[k]); + REQUIRE(d_baseline.tke[k] == d_cxx.tke[k]); + REQUIRE(d_baseline.thetal[k] == d_cxx.thetal[k]); + REQUIRE(d_baseline.qw[k] == d_cxx.qw[k]); + REQUIRE(d_baseline.u_wind[k] == d_cxx.u_wind[k]); + REQUIRE(d_baseline.v_wind[k] == d_cxx.v_wind[k]); + REQUIRE(d_baseline.wthv_sec[k] == d_cxx.wthv_sec[k]); + REQUIRE(d_baseline.tk[k] == d_cxx.tk[k]); + REQUIRE(d_baseline.shoc_ql[k] == d_cxx.shoc_ql[k]); + REQUIRE(d_baseline.shoc_cldfrac[k] == d_cxx.shoc_cldfrac[k]); + REQUIRE(d_baseline.shoc_mix[k] == d_cxx.shoc_mix[k]); + REQUIRE(d_baseline.isotropy[k] == d_cxx.isotropy[k]); + REQUIRE(d_baseline.w_sec[k] == d_cxx.w_sec[k]); + REQUIRE(d_baseline.wqls_sec[k] == d_cxx.wqls_sec[k]); + REQUIRE(d_baseline.brunt[k] == d_cxx.brunt[k]); + REQUIRE(d_baseline.shoc_ql2[k] == d_cxx.shoc_ql2[k]); } - REQUIRE(d_f90.total(d_f90.qtracers) == d_cxx.total(d_cxx.qtracers)); - for (Int k = 0; k < d_f90.total(d_f90.qtracers); ++k) { - REQUIRE(d_f90.qtracers[k] == d_cxx.qtracers[k]); + REQUIRE(d_baseline.total(d_baseline.qtracers) == d_cxx.total(d_cxx.qtracers)); + for (Int k = 0; k < d_baseline.total(d_baseline.qtracers); ++k) { + REQUIRE(d_baseline.qtracers[k] == d_cxx.qtracers[k]); } - REQUIRE(d_f90.total(d_f90.pblh) == d_cxx.total(d_cxx.pblh)); - for (Int k = 0; k < d_f90.total(d_f90.pblh); ++k) { - REQUIRE(d_f90.pblh[k] == d_cxx.pblh[k]); + REQUIRE(d_baseline.total(d_baseline.pblh) == d_cxx.total(d_cxx.pblh)); + for (Int k = 0; k < d_baseline.total(d_baseline.pblh); ++k) { + REQUIRE(d_baseline.pblh[k] == d_cxx.pblh[k]); } - REQUIRE(d_f90.total(d_f90.thl_sec) == d_cxx.total(d_cxx.thl_sec)); - REQUIRE(d_f90.total(d_f90.thl_sec) == d_cxx.total(d_cxx.qw_sec)); - REQUIRE(d_f90.total(d_f90.thl_sec) == d_cxx.total(d_cxx.qwthl_sec)); - REQUIRE(d_f90.total(d_f90.thl_sec) == d_cxx.total(d_cxx.wthl_sec)); - REQUIRE(d_f90.total(d_f90.thl_sec) == d_cxx.total(d_cxx.wqw_sec)); - REQUIRE(d_f90.total(d_f90.thl_sec) == d_cxx.total(d_cxx.wtke_sec)); - REQUIRE(d_f90.total(d_f90.thl_sec) == d_cxx.total(d_cxx.uw_sec)); - REQUIRE(d_f90.total(d_f90.thl_sec) == d_cxx.total(d_cxx.vw_sec)); - REQUIRE(d_f90.total(d_f90.thl_sec) == d_cxx.total(d_cxx.w3)); - for (Int k = 0; k < d_f90.total(d_f90.thl_sec); ++k) { - REQUIRE(d_f90.thl_sec[k] == d_cxx.thl_sec[k]); - REQUIRE(d_f90.qw_sec[k] == d_cxx.qw_sec[k]); - REQUIRE(d_f90.qwthl_sec[k] == d_cxx.qwthl_sec[k]); - REQUIRE(d_f90.wthl_sec[k] == d_cxx.wthl_sec[k]); - REQUIRE(d_f90.wqw_sec[k] == d_cxx.wqw_sec[k]); - REQUIRE(d_f90.wtke_sec[k] == d_cxx.wtke_sec[k]); - REQUIRE(d_f90.uw_sec[k] == d_cxx.uw_sec[k]); - REQUIRE(d_f90.vw_sec[k] == d_cxx.vw_sec[k]); - REQUIRE(d_f90.w3[k] == d_cxx.w3[k]); + REQUIRE(d_baseline.total(d_baseline.thl_sec) == d_cxx.total(d_cxx.thl_sec)); + REQUIRE(d_baseline.total(d_baseline.thl_sec) == d_cxx.total(d_cxx.qw_sec)); + REQUIRE(d_baseline.total(d_baseline.thl_sec) == d_cxx.total(d_cxx.qwthl_sec)); + REQUIRE(d_baseline.total(d_baseline.thl_sec) == d_cxx.total(d_cxx.wthl_sec)); + REQUIRE(d_baseline.total(d_baseline.thl_sec) == d_cxx.total(d_cxx.wqw_sec)); + REQUIRE(d_baseline.total(d_baseline.thl_sec) == d_cxx.total(d_cxx.wtke_sec)); + REQUIRE(d_baseline.total(d_baseline.thl_sec) == d_cxx.total(d_cxx.uw_sec)); + REQUIRE(d_baseline.total(d_baseline.thl_sec) == d_cxx.total(d_cxx.vw_sec)); + REQUIRE(d_baseline.total(d_baseline.thl_sec) == d_cxx.total(d_cxx.w3)); + for (Int k = 0; k < d_baseline.total(d_baseline.thl_sec); ++k) { + REQUIRE(d_baseline.thl_sec[k] == d_cxx.thl_sec[k]); + REQUIRE(d_baseline.qw_sec[k] == d_cxx.qw_sec[k]); + REQUIRE(d_baseline.qwthl_sec[k] == d_cxx.qwthl_sec[k]); + REQUIRE(d_baseline.wthl_sec[k] == d_cxx.wthl_sec[k]); + REQUIRE(d_baseline.wqw_sec[k] == d_cxx.wqw_sec[k]); + REQUIRE(d_baseline.wtke_sec[k] == d_cxx.wtke_sec[k]); + REQUIRE(d_baseline.uw_sec[k] == d_cxx.uw_sec[k]); + REQUIRE(d_baseline.vw_sec[k] == d_cxx.vw_sec[k]); + REQUIRE(d_baseline.w3[k] == d_cxx.w3[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } } // run_bfb }; @@ -495,14 +479,14 @@ TEST_CASE("shoc_main_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocMain; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_main_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocMain; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_mix_length_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_mix_length_tests.cpp index 4d6b5dfdce1c..88336e52ea4f 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_mix_length_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_mix_length_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestCompShocMixLength { +struct UnitWrap::UnitTest::TestCompShocMixLength : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 3; static constexpr Int nlev = 5; @@ -91,13 +91,7 @@ struct UnitWrap::UnitTest::TestCompShocMixLength { } // Call the C++ implementation - SDS.transpose(); - // expects data in fortran layout - compute_shoc_mix_shoc_length_f(SDS.nlev, SDS.shcol, - SDS.tke, SDS.brunt, - SDS.zt_grid, - SDS.l_inf, SDS.shoc_mix); - SDS.transpose(); + compute_shoc_mix_shoc_length(SDS); // Check the results for(Int s = 0; s < shcol; ++s) { @@ -120,11 +114,11 @@ struct UnitWrap::UnitTest::TestCompShocMixLength { } } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - ComputeShocMixShocLengthData SDS_f90[] = { + ComputeShocMixShocLengthData SDS_baseline[] = { // shcol, nlev ComputeShocMixShocLengthData(10, 71), ComputeShocMixShocLengthData(10, 12), @@ -133,48 +127,49 @@ struct UnitWrap::UnitTest::TestCompShocMixLength { }; // Generate random input data - for (auto& d : SDS_f90) { + for (auto& d : SDS_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state ComputeShocMixShocLengthData SDS_cxx[] = { - ComputeShocMixShocLengthData(SDS_f90[0]), - ComputeShocMixShocLengthData(SDS_f90[1]), - ComputeShocMixShocLengthData(SDS_f90[2]), - ComputeShocMixShocLengthData(SDS_f90[3]), + ComputeShocMixShocLengthData(SDS_baseline[0]), + ComputeShocMixShocLengthData(SDS_baseline[1]), + ComputeShocMixShocLengthData(SDS_baseline[2]), + ComputeShocMixShocLengthData(SDS_baseline[3]), }; + static constexpr Int num_runs = sizeof(SDS_baseline) / sizeof(ComputeShocMixShocLengthData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : SDS_f90) { - // expects data in C layout - compute_shoc_mix_shoc_length(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : SDS_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : SDS_cxx) { - d.transpose(); - // expects data in fortran layout - compute_shoc_mix_shoc_length_f(d.nlev, d.shcol, - d.tke, d.brunt, - d.zt_grid, - d.l_inf, d.shoc_mix); - d.transpose(); + compute_shoc_mix_shoc_length(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(SDS_f90) / sizeof(ComputeShocMixShocLengthData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - ComputeShocMixShocLengthData& d_f90 = SDS_f90[i]; + ComputeShocMixShocLengthData& d_baseline = SDS_baseline[i]; ComputeShocMixShocLengthData& d_cxx = SDS_cxx[i]; - for (Int k = 0; k < d_f90.total(d_f90.shoc_mix); ++k) { - REQUIRE(d_f90.shoc_mix[k] == d_cxx.shoc_mix[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.shoc_mix); ++k) { + REQUIRE(d_baseline.shoc_mix[k] == d_cxx.shoc_mix[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + SDS_cxx[i].write(Base::m_fid); + } } } }; @@ -189,14 +184,14 @@ TEST_CASE("shoc_mix_length_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestCompShocMixLength; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_mix_length_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestCompShocMixLength; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_omega_diag_third_moms_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_omega_diag_third_moms_tests.cpp deleted file mode 100644 index e7fd4a507ae2..000000000000 --- a/components/eamxx/src/physics/shoc/tests/shoc_omega_diag_third_moms_tests.cpp +++ /dev/null @@ -1,106 +0,0 @@ -#include "catch2/catch.hpp" - -#include "shoc_unit_tests_common.hpp" -#include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" -#include "physics/share/physics_constants.hpp" -#include "share/scream_types.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/util/ekat_arch.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" - -#include -#include -#include -#include - -namespace scream { -namespace shoc { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestOmegadiagThirdMoms { - - static void run_property() - { - - // Tests for the SHOC function: - // omega_terms_diag_third_shoc_moment - - // TEST ONE - // Call the function twice, with two different sets of terms. - // In the second test the fterms should be larger. Verify - // that the omega0 and omega1 terms are the same, but that the - // omega2 term has increased. - - // buoyancy term (isotropy squared * brunt vaisalla frequency) - constexpr static Real buoy_sgs2 = -100; - // f3 term - constexpr static Real f3_test1a = 14; - // f4 term - constexpr static Real f4_test1a = 5; - - // Initialize data structure for bridging to F90 - OmegaTermsDiagThirdShocMomentData SDS; - - // Load up the data - SDS.buoy_sgs2 = buoy_sgs2; - SDS.f3 = f3_test1a; - SDS.f4 = f4_test1a; - - // Call the fortran implementation - omega_terms_diag_third_shoc_moment(SDS); - - // Save test results - Real omega0_test1a = SDS.omega0; - Real omega1_test1a = SDS.omega1; - Real omega2_test1a = SDS.omega2; - - // Now load up data for second part of test - // Feed in LARGER values - SDS.f3 = 2*f3_test1a; - SDS.f4 = 2*f4_test1a; - - // Call the fortran implementation - omega_terms_diag_third_shoc_moment(SDS); - - // Now check the result - - // omega0 and omega1 should NOT have changed - REQUIRE(SDS.omega0 == omega0_test1a); - REQUIRE(SDS.omega1 == omega1_test1a); - - // omega2 should have increased - REQUIRE(SDS.omega2 > omega2_test1a); - - } - - static void run_bfb() - { - // TODO - } - -}; - -} // namespace unit_test -} // namespace shoc -} // namespace scream - -namespace{ - -TEST_CASE("shoc_omega_diag_third_moms_property", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestOmegadiagThirdMoms; - - TestStruct::run_property(); -} - -TEST_CASE("shoc_omega_diag_third_moms_bfb", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestOmegadiagThirdMoms; - - TestStruct::run_bfb(); -} - -} // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_check_pblh_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_check_pblh_tests.cpp index 2d5f01c8ff27..02489c97fd38 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_check_pblh_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_check_pblh_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "shoc_unit_tests_common.hpp" @@ -14,9 +14,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestPblintdCheckPblh { +struct UnitWrap::UnitTest::TestPblintdCheckPblh : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr auto ustar_min = scream::shoc::Constants::ustar_min; static constexpr Int shcol = 5; @@ -68,9 +68,7 @@ struct UnitWrap::UnitTest::TestPblintdCheckPblh { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - pblintd_check_pblh_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.nlev, SDS.z, SDS.ustar, SDS.check, SDS.pblh); - SDS.transpose(); // go back to C layout + pblintd_check_pblh(SDS); // Check the result // Check that PBL height is greater than zero. This is an @@ -80,11 +78,11 @@ struct UnitWrap::UnitTest::TestPblintdCheckPblh { } } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - PblintdCheckPblhData f90_data[] = { + PblintdCheckPblhData baseline_data[] = { PblintdCheckPblhData(36, 72, 73), PblintdCheckPblhData(72, 72, 73), PblintdCheckPblhData(128, 72, 73), @@ -92,46 +90,50 @@ struct UnitWrap::UnitTest::TestPblintdCheckPblh { }; // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { + // Alternatively, you can use the baseline_data construtors/initializer lists to hardcode data + for (auto& d : baseline_data) { d.randomize(engine, { {d.check, {1, 1}} }); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state PblintdCheckPblhData cxx_data[] = { - PblintdCheckPblhData(f90_data[0]), - PblintdCheckPblhData(f90_data[1]), - PblintdCheckPblhData(f90_data[2]), - PblintdCheckPblhData(f90_data[3]), + PblintdCheckPblhData(baseline_data[0]), + PblintdCheckPblhData(baseline_data[1]), + PblintdCheckPblhData(baseline_data[2]), + PblintdCheckPblhData(baseline_data[3]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - pblintd_check_pblh(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - pblintd_check_pblh_f(d.shcol, d.nlev, d.nlevi, d.nlev, d.z, d.ustar, d.check, d.pblh); - d.transpose(); // go back to C layout + pblintd_check_pblh(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(PblintdCheckPblhData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(PblintdCheckPblhData); for (Int i = 0; i < num_runs; ++i) { - PblintdCheckPblhData& d_f90 = f90_data[i]; + PblintdCheckPblhData& d_baseline = baseline_data[i]; PblintdCheckPblhData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.pblh); ++k) { - REQUIRE(d_f90.total(d_f90.pblh) == d_cxx.total(d_cxx.pblh)); - REQUIRE(d_f90.pblh[k] == d_cxx.pblh[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.pblh); ++k) { + REQUIRE(d_baseline.total(d_baseline.pblh) == d_cxx.total(d_cxx.pblh)); + REQUIRE(d_baseline.pblh[k] == d_cxx.pblh[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } } // run_bfb @@ -147,14 +149,14 @@ TEST_CASE("pblintd_check_pblh_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestPblintdCheckPblh; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("pblintd_check_pblh_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestPblintdCheckPblh; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_cldcheck_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_cldcheck_tests.cpp index df65050acacc..a0beea7e4ea7 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_cldcheck_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_cldcheck_tests.cpp @@ -3,7 +3,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -22,9 +22,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestPblintdCldCheck { +struct UnitWrap::UnitTest::TestPblintdCldCheck : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 5; static constexpr Int nlev = 3; @@ -80,8 +80,7 @@ struct UnitWrap::UnitTest::TestPblintdCldCheck { } // Call the C++ implementation - SDS.transpose(); - shoc_pblintd_cldcheck_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.zi, SDS.cldn, SDS.pblh); + pblintd_cldcheck(SDS); // Check the result for(Int s = 0; s < shcol; ++s) { @@ -94,11 +93,11 @@ struct UnitWrap::UnitTest::TestPblintdCldCheck { } // run_property - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - PblintdCldcheckData cldcheck_data_f90[] = { + PblintdCldcheckData cldcheck_data_baseline[] = { // shcol, nlev, nlevi PblintdCldcheckData(36, 128, 129), PblintdCldcheckData(72, 128, 129), @@ -106,35 +105,41 @@ struct UnitWrap::UnitTest::TestPblintdCldCheck { PblintdCldcheckData(256, 128, 129), }; - for (auto& d : cldcheck_data_f90) { + for (auto& d : cldcheck_data_baseline) { d.randomize(engine); } PblintdCldcheckData cldcheck_data_cxx[] = { - PblintdCldcheckData(cldcheck_data_f90[0]), - PblintdCldcheckData(cldcheck_data_f90[1]), - PblintdCldcheckData(cldcheck_data_f90[2]), - PblintdCldcheckData(cldcheck_data_f90[3]), + PblintdCldcheckData(cldcheck_data_baseline[0]), + PblintdCldcheckData(cldcheck_data_baseline[1]), + PblintdCldcheckData(cldcheck_data_baseline[2]), + PblintdCldcheckData(cldcheck_data_baseline[3]), }; - for (auto& d : cldcheck_data_f90) { - // expects data in C layout - pblintd_cldcheck(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : cldcheck_data_baseline) { + d.read(Base::m_fid); + } } for (auto& d : cldcheck_data_cxx) { - d.transpose(); - shoc_pblintd_cldcheck_f(d.shcol, d.nlev, d.nlevi, d.zi, d.cldn, d.pblh); + pblintd_cldcheck(d); } - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(cldcheck_data_f90) / sizeof(PblintdCldcheckData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(cldcheck_data_baseline) / sizeof(PblintdCldcheckData); for (Int i = 0; i < num_runs; ++i) { const Int shcol = cldcheck_data_cxx[i].shcol; for (Int k = 0; k < shcol; ++k) { - REQUIRE(cldcheck_data_f90[i].pblh[k] == cldcheck_data_cxx[i].pblh[k]); + REQUIRE(cldcheck_data_baseline[i].pblh[k] == cldcheck_data_cxx[i].pblh[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cldcheck_data_cxx) { + d.write(Base::m_fid); + } } } // run_bfb @@ -149,14 +154,14 @@ namespace { TEST_CASE("shoc_pblintd_cldcheck_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestPblintdCldCheck; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_pblintd_cldcheck_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestPblintdCldCheck; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_height_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_height_tests.cpp index 9c33c7de68be..1a44d74782f1 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_height_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_height_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "shoc_unit_tests_common.hpp" @@ -14,9 +14,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestPblintdHeight { +struct UnitWrap::UnitTest::TestPblintdHeight : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr auto ustar_min = scream::shoc::Constants::ustar_min; static const auto approx_zero = Approx(0.0).margin(1e-16); @@ -86,10 +86,7 @@ struct UnitWrap::UnitTest::TestPblintdHeight { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - pblintd_height_f(SDS.shcol, SDS.nlev, SDS.npbl, SDS.z, SDS.u, SDS.v, SDS.ustar, - SDS.thv, SDS.thv_ref, SDS.pblh, SDS.rino, SDS.check); - SDS.transpose(); // go back to C layout + pblintd_height(SDS); // Check the result for(Int s = 0; s < shcol; ++s) { @@ -123,10 +120,7 @@ struct UnitWrap::UnitTest::TestPblintdHeight { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - pblintd_height_f(SDS.shcol, SDS.nlev, SDS.npbl, SDS.z, SDS.u, SDS.v, SDS.ustar, - SDS.thv, SDS.thv_ref, SDS.pblh, SDS.rino, SDS.check); - SDS.transpose(); // go back to C layout + pblintd_height(SDS); // Check the result for(Int s = 0; s < shcol; ++s) { @@ -165,10 +159,7 @@ struct UnitWrap::UnitTest::TestPblintdHeight { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - pblintd_height_f(SDS.shcol, SDS.nlev, SDS.npbl, SDS.z, SDS.u, SDS.v, SDS.ustar, - SDS.thv, SDS.thv_ref, SDS.pblh, SDS.rino, SDS.check); - SDS.transpose(); // go back to C layout + pblintd_height(SDS); // Check that PBLH is zero (not modified) everywhere for(Int s = 0; s < shcol; ++s) { @@ -177,13 +168,13 @@ struct UnitWrap::UnitTest::TestPblintdHeight { } // run_property - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); Int npbl_rand = rand()%72 + 1; - PblintdHeightData f90_data[] = { + PblintdHeightData baseline_data[] = { PblintdHeightData(10, 72, 1), PblintdHeightData(10, 72, 72), PblintdHeightData(10, 72, npbl_rand), @@ -193,49 +184,53 @@ struct UnitWrap::UnitTest::TestPblintdHeight { }; // Generate random input data - for (auto& d : f90_data) { + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state PblintdHeightData cxx_data[] = { - PblintdHeightData(f90_data[0]), - PblintdHeightData(f90_data[1]), - PblintdHeightData(f90_data[2]), - PblintdHeightData(f90_data[3]), - PblintdHeightData(f90_data[4]), - PblintdHeightData(f90_data[5]), + PblintdHeightData(baseline_data[0]), + PblintdHeightData(baseline_data[1]), + PblintdHeightData(baseline_data[2]), + PblintdHeightData(baseline_data[3]), + PblintdHeightData(baseline_data[4]), + PblintdHeightData(baseline_data[5]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - pblintd_height(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - pblintd_height_f(d.shcol, d.nlev, d.npbl, d.z, d.u, d.v, d.ustar, d.thv, d.thv_ref, d.pblh, d.rino, d.check); - d.transpose(); // go back to C layout + pblintd_height(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(PblintdHeightData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(PblintdHeightData); for (Int i = 0; i < num_runs; ++i) { - PblintdHeightData& d_f90 = f90_data[i]; + PblintdHeightData& d_baseline = baseline_data[i]; PblintdHeightData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.pblh); ++k) { - REQUIRE(d_f90.total(d_f90.pblh) == d_cxx.total(d_cxx.pblh)); - REQUIRE(d_f90.pblh[k] == d_cxx.pblh[k]); - REQUIRE(d_f90.total(d_f90.pblh) == d_cxx.total(d_cxx.check)); - REQUIRE(d_f90.check[k] == d_cxx.check[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.pblh); ++k) { + REQUIRE(d_baseline.total(d_baseline.pblh) == d_cxx.total(d_cxx.pblh)); + REQUIRE(d_baseline.pblh[k] == d_cxx.pblh[k]); + REQUIRE(d_baseline.total(d_baseline.pblh) == d_cxx.total(d_cxx.check)); + REQUIRE(d_baseline.check[k] == d_cxx.check[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } } // run_bfb @@ -251,14 +246,14 @@ TEST_CASE("pblintd_height_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestPblintdHeight; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("pblintd_height_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestPblintdHeight; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_init_pot_test.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_init_pot_test.cpp index b01c39eed95a..9f7c2d65142c 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_init_pot_test.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_init_pot_test.cpp @@ -3,7 +3,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -22,9 +22,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestPblintdInitPot { +struct UnitWrap::UnitTest::TestPblintdInitPot : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; static constexpr Int nlev = 1; @@ -77,7 +77,7 @@ struct UnitWrap::UnitTest::TestPblintdInitPot { } // call the C++ implementation - shoc_pblintd_init_pot_f(SDS.shcol, SDS.nlev, SDS.thl, SDS.ql, SDS.q, SDS.thv); + pblintd_init_pot(SDS); // Check the result. // Verify that virtual potential temperature is idential @@ -126,7 +126,7 @@ struct UnitWrap::UnitTest::TestPblintdInitPot { } // Call the C++ implementation - shoc_pblintd_init_pot_f(SDS.shcol, SDS.nlev, SDS.thl, SDS.ql, SDS.q, SDS.thv); + pblintd_init_pot(SDS); // Check test // Verify that column with condensate loading @@ -149,11 +149,11 @@ struct UnitWrap::UnitTest::TestPblintdInitPot { } // run_property - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - PblintdInitPotData pblintd_init_pot_data_f90[] = { + PblintdInitPotData pblintd_init_pot_data_baseline[] = { // shcol, nlev PblintdInitPotData(36, 72), PblintdInitPotData(72, 72), @@ -161,37 +161,44 @@ struct UnitWrap::UnitTest::TestPblintdInitPot { PblintdInitPotData(256, 72), }; - for (auto& d : pblintd_init_pot_data_f90) { + for (auto& d : pblintd_init_pot_data_baseline) { d.randomize(engine); } PblintdInitPotData pblintd_init_pot_data_cxx[] = { - PblintdInitPotData(pblintd_init_pot_data_f90[0]), - PblintdInitPotData(pblintd_init_pot_data_f90[1]), - PblintdInitPotData(pblintd_init_pot_data_f90[2]), - PblintdInitPotData(pblintd_init_pot_data_f90[3]), + PblintdInitPotData(pblintd_init_pot_data_baseline[0]), + PblintdInitPotData(pblintd_init_pot_data_baseline[1]), + PblintdInitPotData(pblintd_init_pot_data_baseline[2]), + PblintdInitPotData(pblintd_init_pot_data_baseline[3]), }; - for (auto& d : pblintd_init_pot_data_f90) { - // expects data in C layout - pblintd_init_pot(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : pblintd_init_pot_data_baseline) { + d.read(Base::m_fid); + } } for (auto& d : pblintd_init_pot_data_cxx) { - shoc_pblintd_init_pot_f(d.shcol, d.nlev, d.thl, d.ql, d.q, d.thv); + pblintd_init_pot(d); } - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(pblintd_init_pot_data_f90) / sizeof(PblintdInitPotData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(pblintd_init_pot_data_baseline) / sizeof(PblintdInitPotData); for (Int i = 0; i < num_runs; ++i) { Int shcol = pblintd_init_pot_data_cxx[i].shcol; Int nlev = pblintd_init_pot_data_cxx[i].nlev; for (Int j = 0; j < shcol; ++j ) { for (Int k = 0; k < nlev; ++k) { - REQUIRE(pblintd_init_pot_data_f90[i].thv[j*k] == pblintd_init_pot_data_cxx[i].thv[j*k]); + REQUIRE(pblintd_init_pot_data_baseline[i].thv[j*k] == pblintd_init_pot_data_cxx[i].thv[j*k]); } } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : pblintd_init_pot_data_cxx) { + d.write(Base::m_fid); + } } } @@ -207,14 +214,14 @@ TEST_CASE("shoc_pblintd_init_pot_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestPblintdInitPot; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_pblintd_init_pot_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestPblintdInitPot; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_surf_temp_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_surf_temp_tests.cpp index ccf3d110fc8d..2e876996a6a4 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_surf_temp_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_surf_temp_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "shoc_unit_tests_common.hpp" @@ -14,9 +14,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestPblintdSurfTemp { +struct UnitWrap::UnitTest::TestPblintdSurfTemp : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr auto ustar_min = scream::shoc::Constants::ustar_min; static constexpr Int shcol = 4; @@ -84,10 +84,7 @@ struct UnitWrap::UnitTest::TestPblintdSurfTemp { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - pblintd_surf_temp_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.z, SDS.ustar, SDS.obklen, - SDS.kbfs, SDS.thv, SDS.tlv, SDS.pblh, SDS.check, SDS.rino); - SDS.transpose(); // go back to C layout + pblintd_surf_temp(SDS); // Check the result for(Int s = 0; s < shcol; ++s) { @@ -112,11 +109,11 @@ struct UnitWrap::UnitTest::TestPblintdSurfTemp { } // run_property - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - PblintdSurfTempData f90_data[] = { + PblintdSurfTempData baseline_data[] = { PblintdSurfTempData(6, 7, 8), PblintdSurfTempData(64, 72, 73), PblintdSurfTempData(128, 72, 73), @@ -124,54 +121,58 @@ struct UnitWrap::UnitTest::TestPblintdSurfTemp { }; // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { + // Alternatively, you can use the baseline_data construtors/initializer lists to hardcode data + for (auto& d : baseline_data) { d.randomize(engine, { {d.obklen, {100., 200.}} }); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state PblintdSurfTempData cxx_data[] = { - PblintdSurfTempData(f90_data[0]), - PblintdSurfTempData(f90_data[1]), - PblintdSurfTempData(f90_data[2]), - PblintdSurfTempData(f90_data[3]), + PblintdSurfTempData(baseline_data[0]), + PblintdSurfTempData(baseline_data[1]), + PblintdSurfTempData(baseline_data[2]), + PblintdSurfTempData(baseline_data[3]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - pblintd_surf_temp(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - pblintd_surf_temp_f(d.shcol, d.nlev, d.nlevi, d.z, d.ustar, d.obklen, d.kbfs, d.thv, d.tlv, d.pblh, d.check, d.rino); - d.transpose(); // go back to C layout + pblintd_surf_temp(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(PblintdSurfTempData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(PblintdSurfTempData); for (Int i = 0; i < num_runs; ++i) { - PblintdSurfTempData& d_f90 = f90_data[i]; + PblintdSurfTempData& d_baseline = baseline_data[i]; PblintdSurfTempData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.tlv); ++k) { - REQUIRE(d_f90.total(d_f90.tlv) == d_cxx.total(d_cxx.tlv)); - REQUIRE(d_f90.tlv[k] == d_cxx.tlv[k]); - REQUIRE(d_f90.total(d_f90.tlv) == d_cxx.total(d_cxx.pblh)); - REQUIRE(d_f90.pblh[k] == d_cxx.pblh[k]); - REQUIRE(d_f90.total(d_f90.tlv) == d_cxx.total(d_cxx.check)); - REQUIRE(d_f90.check[k] == d_cxx.check[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.tlv); ++k) { + REQUIRE(d_baseline.total(d_baseline.tlv) == d_cxx.total(d_cxx.tlv)); + REQUIRE(d_baseline.tlv[k] == d_cxx.tlv[k]); + REQUIRE(d_baseline.total(d_baseline.tlv) == d_cxx.total(d_cxx.pblh)); + REQUIRE(d_baseline.pblh[k] == d_cxx.pblh[k]); + REQUIRE(d_baseline.total(d_baseline.tlv) == d_cxx.total(d_cxx.check)); + REQUIRE(d_baseline.check[k] == d_cxx.check[k]); } - for (Int k = 0; k < d_f90.total(d_f90.rino); ++k) { - REQUIRE(d_f90.total(d_f90.rino) == d_cxx.total(d_cxx.rino)); - REQUIRE(d_f90.rino[k] == d_cxx.rino[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.rino); ++k) { + REQUIRE(d_baseline.total(d_baseline.rino) == d_cxx.total(d_cxx.rino)); + REQUIRE(d_baseline.rino[k] == d_cxx.rino[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } } // run_bfb @@ -187,14 +188,14 @@ TEST_CASE("pblintd_surf_temp_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestPblintdSurfTemp; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("pblintd_surf_temp_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestPblintdSurfTemp; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_tests.cpp index b5cd117228bf..dedc534fbd1b 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "shoc_unit_tests_common.hpp" @@ -14,9 +14,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestPblintd { +struct UnitWrap::UnitTest::TestPblintd : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr auto ustar_min = scream::shoc::Constants::ustar_min; static constexpr Int shcol = 5; @@ -122,11 +122,7 @@ struct UnitWrap::UnitTest::TestPblintd { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - pblintd_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.npbl, SDS.z, SDS.zi, - SDS.thl, SDS.ql, SDS.q, SDS.u, SDS.v, SDS.ustar, SDS.obklen, - SDS.kbfs, SDS.cldn, SDS.pblh); - SDS.transpose(); // go back to C layout + pblintd(SDS); // Make sure PBL height is reasonable // Should be larger than second lowest zi level and lower @@ -140,13 +136,13 @@ struct UnitWrap::UnitTest::TestPblintd { } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); Int npbl_rand = rand()%71 + 1; - PblintdData f90_data[] = { + PblintdData baseline_data[] = { PblintdData(10, 71, 72, 71), PblintdData(10, 71, 72, 1), PblintdData(10, 71, 72, npbl_rand), @@ -156,47 +152,51 @@ struct UnitWrap::UnitTest::TestPblintd { }; // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { + // Alternatively, you can use the baseline_data construtors/initializer lists to hardcode data + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state PblintdData cxx_data[] = { - PblintdData(f90_data[0]), - PblintdData(f90_data[1]), - PblintdData(f90_data[2]), - PblintdData(f90_data[3]), - PblintdData(f90_data[4]), - PblintdData(f90_data[5]), + PblintdData(baseline_data[0]), + PblintdData(baseline_data[1]), + PblintdData(baseline_data[2]), + PblintdData(baseline_data[3]), + PblintdData(baseline_data[4]), + PblintdData(baseline_data[5]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - pblintd(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - pblintd_f(d.shcol, d.nlev, d.nlevi, d.npbl, d.z, d.zi, d.thl, d.ql, d.q, d.u, d.v, d.ustar, d.obklen, d.kbfs, d.cldn, d.pblh); - d.transpose(); // go back to C layout + pblintd(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(PblintdData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(PblintdData); for (Int i = 0; i < num_runs; ++i) { - PblintdData& d_f90 = f90_data[i]; + PblintdData& d_baseline = baseline_data[i]; PblintdData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.pblh); ++k) { - REQUIRE(d_f90.pblh[k] == d_cxx.pblh[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.pblh); ++k) { + REQUIRE(d_baseline.pblh[k] == d_cxx.pblh[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } } // run_bfb @@ -212,14 +212,14 @@ TEST_CASE("pblintd_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestPblintd; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("pblintd_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestPblintd; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_buoyflux_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_buoyflux_tests.cpp index 261cc24dc29e..53c91a605c74 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_buoyflux_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_buoyflux_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" @@ -24,7 +24,6 @@ struct UnitWrap::UnitTest::TestShocPdfCompBuoyFlux { static void run_property() { - static constexpr Real epsterm = scream::physics::Constants::ep_2; // Property tests for the SHOC function // shoc_assumed_pdf_compute_buoyancy_flux @@ -53,7 +52,6 @@ struct UnitWrap::UnitTest::TestShocPdfCompBuoyFlux { SDS.wqwsec = wqwsec_dry; SDS.wqls = wqls_dry; SDS.pval = pval; - SDS.epsterm = epsterm; // Call the fortran implementation shoc_assumed_pdf_compute_buoyancy_flux(SDS); diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_cloudvar_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_cloudvar_tests.cpp index 79594e068a13..7fdd57ff005a 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_cloudvar_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_cloudvar_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_liqflux_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_liqflux_tests.cpp index f1dd2e9bc0ee..bee4f5db740c 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_liqflux_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_liqflux_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_qs_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_qs_tests.cpp index 6e430a02f069..4a3e0cf8a6f5 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_qs_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_qs_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_s_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_s_tests.cpp index 0f40ccb63a04..ffdddaa1928d 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_s_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_s_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_sgsliq_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_sgsliq_tests.cpp index 3964cf6e7ed1..af5e87dc418a 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_sgsliq_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_sgsliq_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pdf_computetemp_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pdf_computetemp_tests.cpp index ce4d27dc15a3..f9044517b6a1 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pdf_computetemp_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pdf_computetemp_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" @@ -37,7 +37,7 @@ struct UnitWrap::UnitTest::TestShocPdfComputeTemp { // Input liquid water potential temperature [K] static constexpr Real thl1 = 305; // Input basepressure [Pa] - static constexpr Real basepres = 100000; + static constexpr Real basepres = C::P0; // Input value of pval [Pa] Real pval = 110000; @@ -55,7 +55,6 @@ struct UnitWrap::UnitTest::TestShocPdfComputeTemp { // Fill in data SDS.thl1 = thl1; - SDS.basepres = basepres; SDS.pval = pval; Int num_tests = SDS.pval/abs(presincr); @@ -64,7 +63,7 @@ struct UnitWrap::UnitTest::TestShocPdfComputeTemp { REQUIRE(presincr < 0); // Make sure our starting pressure is greater than // basepres just so we test a range - REQUIRE(SDS.pval > SDS.basepres); + REQUIRE(SDS.pval > basepres); for (Int s = 0; s < num_tests; ++s){ @@ -81,11 +80,11 @@ struct UnitWrap::UnitTest::TestShocPdfComputeTemp { // If pressure is greater than basepressure then // make sure that temperature is greater than thetal - if (SDS.pval > SDS.basepres){ + if (SDS.pval > basepres){ REQUIRE(SDS.tl1 > SDS.thl1); } // otherwise temperature should be less than thetal - else if(SDS.pval < SDS.basepres){ + else if(SDS.pval < basepres){ REQUIRE(SDS.tl1 < SDS.thl1); } // otherwise if they are equal the temperatures diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pdf_inplume_corr_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pdf_inplume_corr_tests.cpp index d1d17fcebeff..a9ffdf242f36 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pdf_inplume_corr_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pdf_inplume_corr_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pdf_qw_parameters_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pdf_qw_parameters_tests.cpp index 5e7b0b8d72a4..0aabe3da33b8 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pdf_qw_parameters_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pdf_qw_parameters_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" @@ -75,6 +75,8 @@ struct UnitWrap::UnitTest::TestShocQwParameters { SDS.w1_2 = w1_2_test1; SDS.skew_w = Skew_w_test1; SDS.a = a_test1; + SDS.rt_tol = 0; + SDS.w_thresh = 0; // Verify input is physical REQUIRE(SDS.sqrtw2 >= 0); diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pdf_thl_parameters_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pdf_thl_parameters_tests.cpp index f00c7b2c2929..96fdbbc2dfc0 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pdf_thl_parameters_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pdf_thl_parameters_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" @@ -50,8 +50,6 @@ struct UnitWrap::UnitTest::TestShocThlParameters { static constexpr Real Skew_w_test1 = 3; // Define fraction of first gaussian static constexpr Real a_test1 = 0.2; - // Define logical - static constexpr bool dothetal_skew = false; // Define reasonable bounds checking for output static constexpr Real thl_bound_low = 200; // [K] @@ -74,7 +72,8 @@ struct UnitWrap::UnitTest::TestShocThlParameters { SDS.w1_2 = w1_2_test1; SDS.skew_w = Skew_w_test1; SDS.a = a_test1; - SDS.dothetal_skew = dothetal_skew; + SDS.thl_tol = 0; + SDS.w_thresh = 0; // Verify input is physical REQUIRE(SDS.sqrtw2 >= 0); diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pdf_tildetoreal_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pdf_tildetoreal_tests.cpp index 3c0f84a6222b..a0728f2e3584 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pdf_tildetoreal_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pdf_tildetoreal_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pdf_vv_parameters_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pdf_vv_parameters_tests.cpp index 2f0abe43e76a..f923e1b47e2a 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pdf_vv_parameters_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pdf_vv_parameters_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" @@ -46,6 +46,7 @@ struct UnitWrap::UnitTest::TestShocVVParameters { SDS.w_first = w_first_sym; SDS.w_sec = w_sec_sym; SDS.w3var = w3var_sym; + SDS.w_tol_sqd = 0; // Verify input is physical REQUIRE(SDS.w_sec >= 0); diff --git a/components/eamxx/src/physics/shoc/tests/shoc_run_and_cmp.cpp b/components/eamxx/src/physics/shoc/tests/shoc_run_and_cmp.cpp index a5198d959800..8c2313082e54 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_run_and_cmp.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_run_and_cmp.cpp @@ -1,5 +1,5 @@ #include "shoc_main_wrap.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "shoc_ic_cases.hpp" #include "share/scream_types.hpp" @@ -61,7 +61,7 @@ struct Baseline { params_.push_back({ic::Factory::standard, repeat, nsteps, ncol, nlev, num_qtracers, nadv, dt}); } - Int generate_baseline (const std::string& filename, bool use_fortran) { + Int generate_baseline (const std::string& filename) { auto fid = ekat::FILEPtr(fopen(filename.c_str(), "w")); EKAT_REQUIRE_MSG( fid, "generate_baseline can't write " << filename); Int nerr = 0; @@ -74,20 +74,17 @@ struct Baseline { // Run reference shoc on this set of parameters. const auto d = ic::Factory::create(ps.ic, ps.ncol, ps.nlev, ps.num_qtracers); set_params(ps, *d); - shoc_init(ps.nlev, use_fortran); if (ps.repeat > 0 && r == -1) { std::cout << "Running SHOC with ni=" << d->shcol << ", nk=" << d->nlev << ", dt=" << d->dtime << ", ts=" << ps.nsteps; - if (!use_fortran) { - std::cout << ", small_packn=" << SCREAM_SMALL_PACK_SIZE; - } + std::cout << ", small_packn=" << SCREAM_SMALL_PACK_SIZE; std::cout << std::endl; } for (int it = 0; it < ps.nsteps; ++it) { - Int current_microsec = shoc_main(*d, use_fortran); + Int current_microsec = shoc_main(*d); if (r != -1 && ps.repeat > 0) { // do not count the "cold" run duration += current_microsec; @@ -107,30 +104,42 @@ struct Baseline { return nerr; } - Int run_and_cmp (const std::string& filename, const double& tol, bool use_fortran) { - auto fid = ekat::FILEPtr(fopen(filename.c_str(), "r")); - EKAT_REQUIRE_MSG( fid, "generate_baseline can't read " << filename); + Int run_and_cmp (const std::string& filename, const double& tol, bool no_baseline) { + ekat::FILEPtr fid; + if (!no_baseline) { + fid = ekat::FILEPtr(fopen(filename.c_str(), "r")); + EKAT_REQUIRE_MSG( fid, "generate_baseline can't read " << filename); + } Int nerr = 0, ne; int case_num = 0; for (auto ps : params_) { case_num++; - // Read the reference impl's data from the baseline file. - const auto d_ref = ic::Factory::create(ps.ic, ps.ncol, ps.nlev, ps.num_qtracers); - set_params(ps, *d_ref); - // Now run a sequence of other impls. This includes the reference - // implementation b/c it's likely we'll want to change it as we go. - { + if (no_baseline) { const auto d = ic::Factory::create(ps.ic, ps.ncol, ps.nlev, ps.num_qtracers); set_params(ps, *d); - shoc_init(ps.nlev, use_fortran); - for (int it = 0; it < ps.nsteps; it++) { - std::cout << "--- checking case # " << case_num << ", timestep # = " << (it+1)*ps.nadv - << " ---\n" << std::flush; - read(fid, d_ref); - shoc_main(*d,use_fortran); - ne = compare(tol, d_ref, d); - if (ne) std::cout << "Ref impl failed.\n"; - nerr += ne; + for (int it=0; it Path to directory containing baselines.\n" " -t Tolerance for relative error.\n" " -s Number of timesteps. Default=10.\n" " -dt Length of timestep. Default=150.\n" " -i Number of columns(ncol). Default=8.\n" " -k Number of vertical levels. Default=72.\n" " -q Number of q tracers. Default=3.\n" - " -n Number of SHOC loops per timestep. Default=15.\n" + " -l Number of SHOC loops per timestep. Default=15.\n" " -r Number of repetitions, implies timing run (generate + no I/O). Default=0.\n"; return 1; } - bool generate = false, use_fortran = false; + bool generate = false, no_baseline = true; scream::Real tol = SCREAM_BFB_TESTING ? 0 : std::numeric_limits::infinity(); Int nsteps = 10; Int dt = 150; @@ -220,9 +231,9 @@ int main (int argc, char** argv) { Int repeat = 0; std::string baseline_fn; std::string device; - for (int i = 1; i < argc-1; ++i) { - if (ekat::argv_matches(argv[i], "-g", "--generate")) generate = true; - if (ekat::argv_matches(argv[i], "-f", "--fortran")) use_fortran = true; + for (int i = 1; i < argc; ++i) { + if (ekat::argv_matches(argv[i], "-g", "--generate")) { generate = true; no_baseline = false; } + if (ekat::argv_matches(argv[i], "-c", "--compare")) { no_baseline = false; } if (ekat::argv_matches(argv[i], "-b", "--baseline-file")) { expect_another_arg(i, argc); ++i; @@ -258,7 +269,7 @@ int main (int argc, char** argv) { ++i; num_qtracers = std::atoi(argv[i]); } - if (ekat::argv_matches(argv[i], "-n", "--nadv")) { + if (ekat::argv_matches(argv[i], "-l", "--nadv")) { expect_another_arg(i, argc); ++i; nadv = std::atoi(argv[i]); @@ -271,51 +282,31 @@ int main (int argc, char** argv) { generate = true; } } - if (std::string(argv[i])=="--ekat-kokkos-device") { - expect_another_arg(i, argc); - ++i; - device = argv[i]; + if (std::string(argv[i])=="--kokkos-device-id=") { + auto tokens = ekat::split(argv[i],"="); + device = tokens[1]; } } - // Decorate baseline name with precision. - baseline_fn += std::to_string(sizeof(scream::Real)); + // Compute full baseline file name with precision. + baseline_fn += "/shoc_run_and_cmp.baseline" + std::to_string(sizeof(scream::Real)); - std::vector args; - for (int i=0; i" was specified, add kokkos - // initialization flag to argv - // Create it outside the if, so its c_str pointer survives - std::string dev_arg; - if (device!="") { - auto is_int = [] (const std::string& s)->bool { - std::istringstream is(s); - int d; - is >> d; - return !is.fail() && is.eof(); - }; - - EKAT_REQUIRE_MSG (is_int(device), "Error! Invalid device specification.\n"); - - if (std::stoi(device) != -1) { - dev_arg = "--kokkos-device-id=" + device; - args.push_back(const_cast(dev_arg.c_str())); - } - } - - scream::initialize_scream_session(args.size(), args.data()); { + scream::initialize_scream_session(argc, argv); + { Baseline bln(nsteps, static_cast(dt), ncol, nlev, num_qtracers, nadv, repeat); if (generate) { std::cout << "Generating to " << baseline_fn << "\n"; - nerr += bln.generate_baseline(baseline_fn, use_fortran); - } else { + nerr += bln.generate_baseline(baseline_fn); + } else if (no_baseline) { + printf("Running with no baseline actions\n"); + nerr += bln.run_and_cmp(baseline_fn, tol, no_baseline); + } + else { printf("Comparing with %s at tol %1.1e\n", baseline_fn.c_str(), tol); - nerr += bln.run_and_cmp(baseline_fn, tol, use_fortran); + nerr += bln.run_and_cmp(baseline_fn, tol, no_baseline); } - } scream::finalize_scream_session(); + } + scream::finalize_scream_session(); return nerr != 0 ? 1 : 0; } diff --git a/components/eamxx/src/physics/shoc/tests/shoc_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_tests.cpp index 1872cf1b93a4..1884c0e34587 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_tests.cpp @@ -26,43 +26,8 @@ TEST_CASE("FortranDataIterator", "shoc") { REQUIRE(static_cast(f.size) == d->shcol); } -TEST_CASE("shoc_init_f", "shoc") { - int nerr = scream::shoc::test_shoc_init(true); - REQUIRE(nerr == 0); -} - -// This helper returns true if we've been asked to generate Python -// plotting scripts, false otherwise. -bool generating_plot_scripts() { - bool gen_plot_scripts = false; - auto& ts = ekat::TestSession::get(); - auto iter = ts.params.find("gen_plot_scripts"); - if (iter != ts.params.end()) { - // Here's val, passed as gen_plot_scripts=val - std::string val = iter->second; - // Low-case the thing. Isn't C++ a friendly language?? - std::transform(val.begin(), val.end(), val.begin(), - [](unsigned char c){ return std::tolower(c); }); - - // Now decide if the value is true or not. Use CMake sensibilities. - gen_plot_scripts = ((val == "1") or (val == "true") or - (val == "yes") or (val == "on")); - } - return gen_plot_scripts; -} - -TEST_CASE("shoc_ic_f", "shoc") { - int nerr = scream::shoc::test_shoc_ic(true, generating_plot_scripts()); - REQUIRE(nerr == 0); -} - -TEST_CASE("shoc_init_c", "shoc") { - int nerr = scream::shoc::test_shoc_init(false); - REQUIRE(nerr == 0); -} - TEST_CASE("shoc_ic_c", "shoc") { - int nerr = scream::shoc::test_shoc_ic(false); + int nerr = scream::shoc::test_shoc_ic(); REQUIRE(nerr == 0); } diff --git a/components/eamxx/src/physics/shoc/tests/shoc_tke_adv_sgs_tke_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_tke_adv_sgs_tke_tests.cpp index ea2c95470d89..165bf302e4c6 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_tke_adv_sgs_tke_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_tke_adv_sgs_tke_tests.cpp @@ -1,7 +1,7 @@ #include "catch2/catch.hpp" #include "shoc_unit_tests_common.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "shoc_functions.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" @@ -22,9 +22,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocAdvSgsTke { +struct UnitWrap::UnitTest::TestShocAdvSgsTke : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Real mintke = scream::shoc::Constants::mintke; static constexpr Real maxtke = scream::shoc::Constants::maxtke; @@ -95,9 +95,7 @@ struct UnitWrap::UnitTest::TestShocAdvSgsTke { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - adv_sgs_tke_f(SDS.nlev, SDS.shcol, SDS.dtime, SDS.shoc_mix, SDS.wthv_sec, SDS.sterm_zt, SDS.tk, SDS.tke, SDS.a_diss); - SDS.transpose(); // go back to C layout + adv_sgs_tke(SDS); // Check to make sure that there has been // TKE growth @@ -162,9 +160,7 @@ struct UnitWrap::UnitTest::TestShocAdvSgsTke { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - adv_sgs_tke_f(SDS.nlev, SDS.shcol, SDS.dtime, SDS.shoc_mix, SDS.wthv_sec, SDS.sterm_zt, SDS.tk, SDS.tke, SDS.a_diss); - SDS.transpose(); // go back to C layout + adv_sgs_tke(SDS); // Check to make sure that the column with // the smallest length scale has larger @@ -194,11 +190,11 @@ struct UnitWrap::UnitTest::TestShocAdvSgsTke { } } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - AdvSgsTkeData f90_data[] = { + AdvSgsTkeData baseline_data[] = { // shcol, nlev AdvSgsTkeData(10, 71, 72), AdvSgsTkeData(10, 12, 13), @@ -207,45 +203,49 @@ struct UnitWrap::UnitTest::TestShocAdvSgsTke { }; // Generate random input data - for (auto& d : f90_data) { + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state AdvSgsTkeData cxx_data[] = { - AdvSgsTkeData(f90_data[0]), - AdvSgsTkeData(f90_data[1]), - AdvSgsTkeData(f90_data[2]), - AdvSgsTkeData(f90_data[3]), + AdvSgsTkeData(baseline_data[0]), + AdvSgsTkeData(baseline_data[1]), + AdvSgsTkeData(baseline_data[2]), + AdvSgsTkeData(baseline_data[3]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - adv_sgs_tke(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - adv_sgs_tke_f(d.nlev, d.shcol, d.dtime, d.shoc_mix, d.wthv_sec, d.sterm_zt, d.tk, d.tke, d.a_diss); - d.transpose(); // go back to C layout + adv_sgs_tke(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(AdvSgsTkeData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(AdvSgsTkeData); for (Int i = 0; i < num_runs; ++i) { - AdvSgsTkeData& d_f90 = f90_data[i]; + AdvSgsTkeData& d_baseline = baseline_data[i]; AdvSgsTkeData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.tke); ++k) { - REQUIRE(d_f90.tke[k] == d_cxx.tke[k]); - REQUIRE(d_f90.a_diss[k] == d_cxx.a_diss[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.tke); ++k) { + REQUIRE(d_baseline.tke[k] == d_cxx.tke[k]); + REQUIRE(d_baseline.a_diss[k] == d_cxx.a_diss[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } }//run_bfb }; @@ -260,14 +260,14 @@ TEST_CASE("shoc_tke_adv_sgs_tke_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocAdvSgsTke; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_tke_adv_sgs_tke_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocAdvSgsTke; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_tke_column_stab_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_tke_column_stab_tests.cpp index eb68bbc11df1..d234273f4c0a 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_tke_column_stab_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_tke_column_stab_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocIntColStab { +struct UnitWrap::UnitTest::TestShocIntColStab : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; static constexpr Int nlev = 5; @@ -76,9 +76,7 @@ struct UnitWrap::UnitTest::TestShocIntColStab { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - integ_column_stability_f(SDS.nlev, SDS.shcol, SDS.dz_zt, SDS.pres, SDS.brunt, SDS.brunt_int); - SDS.transpose(); // go back to C layout + integ_column_stability(SDS); // Check test // Verify that output is zero @@ -110,9 +108,7 @@ struct UnitWrap::UnitTest::TestShocIntColStab { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - integ_column_stability_f(SDS.nlev, SDS.shcol, SDS.dz_zt, SDS.pres, SDS.brunt, SDS.brunt_int); - SDS.transpose(); // go back to C layout + integ_column_stability(SDS); // Check test // Verify that output is negative @@ -121,12 +117,12 @@ struct UnitWrap::UnitTest::TestShocIntColStab { } } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - //declare data for the f90 function call - IntegColumnStabilityData f90_data[] = { + //declare data for the baseline function call + IntegColumnStabilityData baseline_data[] = { IntegColumnStabilityData(10, 71), IntegColumnStabilityData(10, 12), IntegColumnStabilityData(7, 16), @@ -134,44 +130,48 @@ struct UnitWrap::UnitTest::TestShocIntColStab { }; //Generate random data - for (auto &d : f90_data) { + for (auto &d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data (if any) is in original state IntegColumnStabilityData cxx_data[] = { - IntegColumnStabilityData(f90_data[0]), - IntegColumnStabilityData(f90_data[1]), - IntegColumnStabilityData(f90_data[2]), - IntegColumnStabilityData(f90_data[3]), + IntegColumnStabilityData(baseline_data[0]), + IntegColumnStabilityData(baseline_data[1]), + IntegColumnStabilityData(baseline_data[2]), + IntegColumnStabilityData(baseline_data[3]), }; // Assume all data is in C layout - // Get data from fortran - for (auto &d : f90_data) { - // expects data in C layout - integ_column_stability(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto &d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto &d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - integ_column_stability_f(d.nlev, d.shcol, d.dz_zt, d.pres, d.brunt, d.brunt_int); - d.transpose(); // go back to C layout + integ_column_stability(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(IntegColumnStabilityData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(IntegColumnStabilityData); for (Int i = 0; i < num_runs; ++i) { - IntegColumnStabilityData& d_f90 = f90_data[i]; + IntegColumnStabilityData& d_baseline = baseline_data[i]; IntegColumnStabilityData& d_cxx = cxx_data[i]; - for (Int c = 0; c < d_f90.shcol; ++c) { - REQUIRE(d_f90.brunt_int[c] == d_cxx.brunt_int[c]); + for (Int c = 0; c < d_baseline.shcol; ++c) { + REQUIRE(d_baseline.brunt_int[c] == d_cxx.brunt_int[c]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto &d : cxx_data) { + d.write(Base::m_fid); + } } } //run_bfb }; @@ -186,14 +186,14 @@ TEST_CASE("shoc_tke_column_stab_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocIntColStab; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_tke_column_stab_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocIntColStab; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_tke_isotropic_ts_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_tke_isotropic_ts_tests.cpp index c90128857cbb..b7fba522d8f1 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_tke_isotropic_ts_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_tke_isotropic_ts_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocIsotropicTs { +struct UnitWrap::UnitTest::TestShocIsotropicTs : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Real maxiso = scream::shoc::Constants::maxiso; static constexpr Int shcol = 2; @@ -82,9 +82,7 @@ struct UnitWrap::UnitTest::TestShocIsotropicTs { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - isotropic_ts_f(SDS.nlev, SDS.shcol, SDS.brunt_int, SDS.tke, SDS.a_diss, SDS.brunt, SDS.isotropy); - SDS.transpose(); // go back to C layout + isotropic_ts(SDS); // Check that output falls within reasonable bounds for(Int s = 0; s < shcol; ++s) { @@ -148,9 +146,7 @@ struct UnitWrap::UnitTest::TestShocIsotropicTs { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - isotropic_ts_f(SDS.nlev, SDS.shcol, SDS.brunt_int, SDS.tke, SDS.a_diss, SDS.brunt, SDS.isotropy); - SDS.transpose(); // go back to C layout + isotropic_ts(SDS); // Check that output falls within reasonable bounds for(Int s = 0; s < shcol; ++s) { @@ -178,11 +174,11 @@ struct UnitWrap::UnitTest::TestShocIsotropicTs { } } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - IsotropicTsData f90_data[] = { + IsotropicTsData baseline_data[] = { // shcol, nlev IsotropicTsData(10, 71), IsotropicTsData(10, 12), @@ -191,44 +187,48 @@ struct UnitWrap::UnitTest::TestShocIsotropicTs { }; // Generate random input data - for (auto& d : f90_data) { + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state IsotropicTsData cxx_data[] = { - IsotropicTsData(f90_data[0]), - IsotropicTsData(f90_data[1]), - IsotropicTsData(f90_data[2]), - IsotropicTsData(f90_data[3]), + IsotropicTsData(baseline_data[0]), + IsotropicTsData(baseline_data[1]), + IsotropicTsData(baseline_data[2]), + IsotropicTsData(baseline_data[3]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - isotropic_ts(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - isotropic_ts_f(d.nlev, d.shcol, d.brunt_int, d.tke, d.a_diss, d.brunt, d.isotropy); - d.transpose(); // go back to C layout + isotropic_ts(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(IsotropicTsData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(IsotropicTsData); for (Int i = 0; i < num_runs; ++i) { - IsotropicTsData& d_f90 = f90_data[i]; + IsotropicTsData& d_baseline = baseline_data[i]; IsotropicTsData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.isotropy); ++k) { - REQUIRE(d_f90.isotropy[k] == d_cxx.isotropy[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.isotropy); ++k) { + REQUIRE(d_baseline.isotropy[k] == d_cxx.isotropy[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } }//run_bfb @@ -244,14 +244,14 @@ TEST_CASE("shoc_tke_isotropic_ts_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocIsotropicTs; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_tke_isotropic_ts_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocIsotropicTs; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_tke_shr_prod_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_tke_shr_prod_tests.cpp index 0672ce3b3ae6..00b60a5efdbd 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_tke_shr_prod_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_tke_shr_prod_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocShearProd { +struct UnitWrap::UnitTest::TestShocShearProd : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; static constexpr Int nlev = 5; @@ -94,9 +94,7 @@ struct UnitWrap::UnitTest::TestShocShearProd { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - compute_shr_prod_f(SDS.nlevi, SDS.nlev, SDS.shcol, SDS.dz_zi, SDS.u_wind, SDS.v_wind, SDS.sterm); - SDS.transpose(); // go back to C layout + compute_shr_prod(SDS); // Check test for(Int s = 0; s < shcol; ++s) { @@ -145,9 +143,7 @@ struct UnitWrap::UnitTest::TestShocShearProd { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - compute_shr_prod_f(SDS.nlevi, SDS.nlev, SDS.shcol, SDS.dz_zi, SDS.u_wind, SDS.v_wind, SDS.sterm); - SDS.transpose(); // go back to C layout + compute_shr_prod(SDS); // Check test // Verify that shear term is zero everywhere @@ -159,11 +155,11 @@ struct UnitWrap::UnitTest::TestShocShearProd { } } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - ComputeShrProdData f90_data[] = { + ComputeShrProdData baseline_data[] = { // shcol, nlev ComputeShrProdData(10, 71, 72), ComputeShrProdData(10, 12, 13), @@ -172,44 +168,48 @@ struct UnitWrap::UnitTest::TestShocShearProd { }; // Generate random input data - for (auto& d : f90_data) { + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state ComputeShrProdData cxx_data[] = { - ComputeShrProdData(f90_data[0]), - ComputeShrProdData(f90_data[1]), - ComputeShrProdData(f90_data[2]), - ComputeShrProdData(f90_data[3]), + ComputeShrProdData(baseline_data[0]), + ComputeShrProdData(baseline_data[1]), + ComputeShrProdData(baseline_data[2]), + ComputeShrProdData(baseline_data[3]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - compute_shr_prod(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - compute_shr_prod_f(d.nlevi, d.nlev, d.shcol, d.dz_zi, d.u_wind, d.v_wind, d.sterm); - d.transpose(); // go back to C layout + compute_shr_prod(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(ComputeShrProdData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(ComputeShrProdData); for (Int i = 0; i < num_runs; ++i) { - ComputeShrProdData& d_f90 = f90_data[i]; + ComputeShrProdData& d_baseline = baseline_data[i]; ComputeShrProdData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.sterm); ++k) { - REQUIRE(d_f90.sterm[k] == d_cxx.sterm[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.sterm); ++k) { + REQUIRE(d_baseline.sterm[k] == d_cxx.sterm[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } } //run_bfb }; @@ -224,14 +224,14 @@ TEST_CASE("shoc_tke_shr_prod_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocShearProd; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_tke_shr_prod_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocShearProd; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_tke_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_tke_tests.cpp index f09ecc79c7d3..5273607dd707 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_tke_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_tke_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocTke { +struct UnitWrap::UnitTest::TestShocTke : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Real mintke = scream::shoc::Constants::mintke; static constexpr Real maxtke = scream::shoc::Constants::maxtke; @@ -154,11 +154,7 @@ struct UnitWrap::UnitTest::TestShocTke { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - shoc_tke_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.dtime, SDS.wthv_sec, SDS.shoc_mix, SDS.dz_zi, SDS.dz_zt, - SDS.pres, SDS.tabs, SDS.u_wind, SDS.v_wind, SDS.brunt, SDS.zt_grid, SDS.zi_grid, SDS.pblh, - SDS.tke, SDS.tk, SDS.tkh, SDS.isotropy); - SDS.transpose(); // go back to C layout + shoc_tke(SDS); // Check test // Make sure that TKE has increased everwhere relative @@ -231,11 +227,7 @@ struct UnitWrap::UnitTest::TestShocTke { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - shoc_tke_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.dtime, SDS.wthv_sec, SDS.shoc_mix, SDS.dz_zi, SDS.dz_zt, - SDS.pres, SDS.tabs, SDS.u_wind, SDS.v_wind, SDS.brunt, SDS.zt_grid, SDS.zi_grid, SDS.pblh, - SDS.tke, SDS.tk, SDS.tkh, SDS.isotropy); - SDS.transpose(); // go back to C layout + shoc_tke(SDS); // Check the result @@ -254,11 +246,11 @@ struct UnitWrap::UnitTest::TestShocTke { } } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - ShocTkeData f90_data[] = { + ShocTkeData baseline_data[] = { ShocTkeData(10, 71, 72, 300), ShocTkeData(10, 12, 13, 100), ShocTkeData(7, 16, 17, 50), @@ -266,54 +258,56 @@ struct UnitWrap::UnitTest::TestShocTke { }; // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { + // Alternatively, you can use the baseline_data construtors/initializer lists to hardcode data + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state ShocTkeData cxx_data[] = { - ShocTkeData(f90_data[0]), - ShocTkeData(f90_data[1]), - ShocTkeData(f90_data[2]), - ShocTkeData(f90_data[3]), + ShocTkeData(baseline_data[0]), + ShocTkeData(baseline_data[1]), + ShocTkeData(baseline_data[2]), + ShocTkeData(baseline_data[3]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - shoc_tke(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - shoc_tke_f(d.shcol, d.nlev, d.nlevi, d.dtime, d.wthv_sec, d.shoc_mix, d.dz_zi, d.dz_zt, - d.pres, d.tabs, d.u_wind, d.v_wind, d.brunt, d.zt_grid, d.zi_grid, d.pblh, - d.tke, d.tk, d.tkh, d.isotropy); - d.transpose(); // go back to C layout + shoc_tke(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(ShocTkeData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(ShocTkeData); for (Int i = 0; i < num_runs; ++i) { - ShocTkeData& d_f90 = f90_data[i]; + ShocTkeData& d_baseline = baseline_data[i]; ShocTkeData& d_cxx = cxx_data[i]; - REQUIRE(d_f90.total(d_f90.tke) == d_cxx.total(d_cxx.tke)); - REQUIRE(d_f90.total(d_f90.tke) == d_cxx.total(d_cxx.tk)); - REQUIRE(d_f90.total(d_f90.tke) == d_cxx.total(d_cxx.tkh)); - REQUIRE(d_f90.total(d_f90.tke) == d_cxx.total(d_cxx.isotropy)); - for (Int k = 0; k < d_f90.total(d_f90.tke); ++k) { - REQUIRE(d_f90.tke[k] == d_cxx.tke[k]); - REQUIRE(d_f90.tk[k] == d_cxx.tk[k]); - REQUIRE(d_f90.tkh[k] == d_cxx.tkh[k]); - REQUIRE(d_f90.isotropy[k] == d_cxx.isotropy[k]); + REQUIRE(d_baseline.total(d_baseline.tke) == d_cxx.total(d_cxx.tke)); + REQUIRE(d_baseline.total(d_baseline.tke) == d_cxx.total(d_cxx.tk)); + REQUIRE(d_baseline.total(d_baseline.tke) == d_cxx.total(d_cxx.tkh)); + REQUIRE(d_baseline.total(d_baseline.tke) == d_cxx.total(d_cxx.isotropy)); + for (Int k = 0; k < d_baseline.total(d_baseline.tke); ++k) { + REQUIRE(d_baseline.tke[k] == d_cxx.tke[k]); + REQUIRE(d_baseline.tk[k] == d_cxx.tk[k]); + REQUIRE(d_baseline.tkh[k] == d_cxx.tkh[k]); + REQUIRE(d_baseline.isotropy[k] == d_cxx.isotropy[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } } // run_bfb }; @@ -328,14 +322,14 @@ TEST_CASE("shoc_tke_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocTke; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_tke_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocTke; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } diff --git a/components/eamxx/src/physics/shoc/tests/shoc_unit_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_unit_tests.cpp index 71fe03f3cbbb..f564c74fab4e 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_unit_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_unit_tests.cpp @@ -3,7 +3,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "share/scream_types.hpp" diff --git a/components/eamxx/src/physics/shoc/tests/shoc_update_prognostics_implicit_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_update_prognostics_implicit_tests.cpp index 83cfb8d5707e..175be8429283 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_update_prognostics_implicit_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_update_prognostics_implicit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "shoc_unit_tests_common.hpp" @@ -14,9 +14,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestUpdatePrognosticsImplicit { +struct UnitWrap::UnitTest::TestUpdatePrognosticsImplicit : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 5; static constexpr Int nlev = 5; @@ -259,17 +259,10 @@ struct UnitWrap::UnitTest::TestUpdatePrognosticsImplicit { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - update_prognostics_implicit_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.num_tracer, SDS.dtime, - SDS.dz_zt, SDS.dz_zi, SDS.rho_zt, SDS.zt_grid, SDS.zi_grid, - SDS.tk, SDS.tkh, SDS.uw_sfc, SDS.vw_sfc, SDS.wthl_sfc, SDS.wqw_sfc, - SDS.wtracer_sfc, SDS.thetal, SDS.qw, SDS.tracer, SDS.tke, SDS.u_wind, SDS.v_wind); - SDS.transpose(); // go back to C layout + update_prognostics_implicit(SDS); // Call linear interp to get rho value at surface for checking - SDSL.transpose(); // _f expects data in fortran layout - linear_interp_f(SDSL.x1, SDSL.x2, SDSL.y1, SDSL.y2, SDSL.km1, SDSL.km2, SDSL.ncol, SDSL.minthresh); - SDSL.transpose(); // go back to C layout + linear_interp(SDSL); // Check the result @@ -341,11 +334,11 @@ struct UnitWrap::UnitTest::TestUpdatePrognosticsImplicit { } // run_property - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - UpdatePrognosticsImplicitData f90_data[] = { + UpdatePrognosticsImplicitData baseline_data[] = { UpdatePrognosticsImplicitData(10, 71, 72, 19, .5), UpdatePrognosticsImplicitData(10, 12, 13, 7, .25), UpdatePrognosticsImplicitData(7, 16, 17, 2, .1), @@ -353,62 +346,63 @@ struct UnitWrap::UnitTest::TestUpdatePrognosticsImplicit { }; // Generate random input data - for (auto& d : f90_data) { + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state UpdatePrognosticsImplicitData cxx_data[] = { - UpdatePrognosticsImplicitData(f90_data[0]), - UpdatePrognosticsImplicitData(f90_data[1]), - UpdatePrognosticsImplicitData(f90_data[2]), - UpdatePrognosticsImplicitData(f90_data[3]), + UpdatePrognosticsImplicitData(baseline_data[0]), + UpdatePrognosticsImplicitData(baseline_data[1]), + UpdatePrognosticsImplicitData(baseline_data[2]), + UpdatePrognosticsImplicitData(baseline_data[3]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - update_prognostics_implicit(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - update_prognostics_implicit_f(d.shcol, d.nlev, d.nlevi, d.num_tracer, d.dtime, - d.dz_zt, d.dz_zi, d.rho_zt, d.zt_grid, d.zi_grid, - d.tk, d.tkh, d.uw_sfc, d.vw_sfc, d.wthl_sfc, d.wqw_sfc, - d.wtracer_sfc, d.thetal, d.qw, d.tracer, d.tke, d.u_wind, d.v_wind); - d.transpose(); // go back to C layout + update_prognostics_implicit(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(UpdatePrognosticsImplicitData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(UpdatePrognosticsImplicitData); for (Int i = 0; i < num_runs; ++i) { - UpdatePrognosticsImplicitData& d_f90 = f90_data[i]; + UpdatePrognosticsImplicitData& d_baseline = baseline_data[i]; UpdatePrognosticsImplicitData& d_cxx = cxx_data[i]; - REQUIRE(d_f90.total(d_f90.thetal) == d_cxx.total(d_cxx.thetal)); - REQUIRE(d_f90.total(d_f90.qw) == d_cxx.total(d_cxx.qw)); - REQUIRE(d_f90.total(d_f90.tke) == d_cxx.total(d_cxx.tke)); - REQUIRE(d_f90.total(d_f90.u_wind) == d_cxx.total(d_cxx.u_wind)); - REQUIRE(d_f90.total(d_f90.v_wind) == d_cxx.total(d_cxx.v_wind)); - for (Int k = 0; k < d_f90.total(d_f90.thetal); ++k) { - REQUIRE(d_f90.thetal[k] == d_cxx.thetal[k]); - REQUIRE(d_f90.qw[k] == d_cxx.qw[k]); - REQUIRE(d_f90.tke[k] == d_cxx.tke[k]); - REQUIRE(d_f90.u_wind[k] == d_cxx.u_wind[k]); - REQUIRE(d_f90.v_wind[k] == d_cxx.v_wind[k]); + REQUIRE(d_baseline.total(d_baseline.thetal) == d_cxx.total(d_cxx.thetal)); + REQUIRE(d_baseline.total(d_baseline.qw) == d_cxx.total(d_cxx.qw)); + REQUIRE(d_baseline.total(d_baseline.tke) == d_cxx.total(d_cxx.tke)); + REQUIRE(d_baseline.total(d_baseline.u_wind) == d_cxx.total(d_cxx.u_wind)); + REQUIRE(d_baseline.total(d_baseline.v_wind) == d_cxx.total(d_cxx.v_wind)); + for (Int k = 0; k < d_baseline.total(d_baseline.thetal); ++k) { + REQUIRE(d_baseline.thetal[k] == d_cxx.thetal[k]); + REQUIRE(d_baseline.qw[k] == d_cxx.qw[k]); + REQUIRE(d_baseline.tke[k] == d_cxx.tke[k]); + REQUIRE(d_baseline.u_wind[k] == d_cxx.u_wind[k]); + REQUIRE(d_baseline.v_wind[k] == d_cxx.v_wind[k]); } - REQUIRE(d_f90.total(d_f90.tracer) == d_cxx.total(d_cxx.tracer)); - for (Int k = 0; k < d_f90.total(d_f90.tracer); ++k) { - REQUIRE(d_f90.tracer[k] == d_cxx.tracer[k]); + REQUIRE(d_baseline.total(d_baseline.tracer) == d_cxx.total(d_cxx.tracer)); + for (Int k = 0; k < d_baseline.total(d_baseline.tracer); ++k) { + REQUIRE(d_baseline.tracer[k] == d_cxx.tracer[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } } // run_bfb @@ -424,14 +418,14 @@ TEST_CASE("update_prognostics_implicit_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestUpdatePrognosticsImplicit; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("update_prognostics_implicit_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestUpdatePrognosticsImplicit; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_varorcovar_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_varorcovar_tests.cpp index 4a743009f244..35d60abfa5fa 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_varorcovar_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_varorcovar_tests.cpp @@ -4,7 +4,7 @@ #include "physics/share/physics_constants.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -23,9 +23,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocVarorCovar { +struct UnitWrap::UnitTest::TestShocVarorCovar : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; static constexpr Int nlev = 4; @@ -116,13 +116,7 @@ struct UnitWrap::UnitTest::TestShocVarorCovar { } // Call the C++ implementation for variance - SDS.transpose(); - // expects data in fortran layout - calc_shoc_varorcovar_f(SDS.shcol, SDS.nlev, SDS.nlevi, - SDS.tunefac, SDS.isotropy_zi, - SDS.tkh_zi, SDS.dz_zi, - SDS.invar1, SDS.invar2, SDS.varorcovar); - SDS.transpose(); + calc_shoc_varorcovar(SDS); // Check the results for(Int s = 0; s < shcol; ++s) { @@ -179,13 +173,7 @@ struct UnitWrap::UnitTest::TestShocVarorCovar { } // Call the C++ implementation for covariance - SDS.transpose(); - // expects data in fortran layout - calc_shoc_varorcovar_f(SDS.shcol, SDS.nlev, SDS.nlevi, - SDS.tunefac, SDS.isotropy_zi, - SDS.tkh_zi, SDS.dz_zi, - SDS.invar1, SDS.invar2, SDS.varorcovar); - SDS.transpose(); + calc_shoc_varorcovar(SDS); // Check the results for(Int s = 0; s < shcol; ++s) { @@ -263,13 +251,7 @@ struct UnitWrap::UnitTest::TestShocVarorCovar { } // Call the C++ implementation for variance - SDS.transpose(); - // expects data in fortran layout - calc_shoc_varorcovar_f(SDS.shcol, SDS.nlev, SDS.nlevi, - SDS.tunefac, SDS.isotropy_zi, - SDS.tkh_zi, SDS.dz_zi, - SDS.invar1, SDS.invar2, SDS.varorcovar); - SDS.transpose(); + calc_shoc_varorcovar(SDS); // Check the results for(Int s = 0; s < shcol; ++s) { @@ -284,11 +266,11 @@ struct UnitWrap::UnitTest::TestShocVarorCovar { } } -static void run_bfb() +void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - CalcShocVarorcovarData SDS_f90[] = { + CalcShocVarorcovarData SDS_baseline[] = { // shcol, nlev, nlevi, tunefac CalcShocVarorcovarData(10, 71, 72, 1), CalcShocVarorcovarData(10, 12, 13, 1), @@ -297,48 +279,49 @@ static void run_bfb() }; // Generate random input data - for (auto& d : SDS_f90) { + for (auto& d : SDS_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state CalcShocVarorcovarData SDS_cxx[] = { - CalcShocVarorcovarData(SDS_f90[0]), - CalcShocVarorcovarData(SDS_f90[1]), - CalcShocVarorcovarData(SDS_f90[2]), - CalcShocVarorcovarData(SDS_f90[3]), + CalcShocVarorcovarData(SDS_baseline[0]), + CalcShocVarorcovarData(SDS_baseline[1]), + CalcShocVarorcovarData(SDS_baseline[2]), + CalcShocVarorcovarData(SDS_baseline[3]), }; + static constexpr Int num_runs = sizeof(SDS_baseline) / sizeof(CalcShocVarorcovarData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : SDS_f90) { - // expects data in C layout - calc_shoc_varorcovar(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : SDS_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : SDS_cxx) { - d.transpose(); - // expects data in fortran layout - calc_shoc_varorcovar_f(d.shcol, d.nlev, d.nlevi, - d.tunefac, d.isotropy_zi, - d.tkh_zi, d.dz_zi, - d.invar1, d.invar2, d.varorcovar); - d.transpose(); + calc_shoc_varorcovar(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(SDS_f90) / sizeof(CalcShocVarorcovarData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - CalcShocVarorcovarData& d_f90 = SDS_f90[i]; + CalcShocVarorcovarData& d_baseline = SDS_baseline[i]; CalcShocVarorcovarData& d_cxx = SDS_cxx[i]; - for (Int k = 0; k < d_f90.total(d_f90.varorcovar); ++k) { - REQUIRE(d_f90.varorcovar[k] == d_cxx.varorcovar[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.varorcovar); ++k) { + REQUIRE(d_baseline.varorcovar[k] == d_cxx.varorcovar[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + SDS_cxx[i].write(Base::m_fid); + } } } }; @@ -353,14 +336,14 @@ TEST_CASE("shoc_varorcovar_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocVarorCovar; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_varorcovar_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocVarorCovar; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_vd_shoc_decomp_and_solve_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_vd_shoc_decomp_and_solve_tests.cpp index 3bd213cf7840..27326a0df167 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_vd_shoc_decomp_and_solve_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_vd_shoc_decomp_and_solve_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "shoc_unit_tests_common.hpp" @@ -14,13 +14,13 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestVdShocDecompandSolve { +struct UnitWrap::UnitTest::TestVdShocDecompandSolve : public UnitWrap::UnitTest::Base { - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - VdShocDecompandSolveData f90_data[] = { + VdShocDecompandSolveData baseline_data[] = { // shcol, nlev, nlevi, dtime, n_rhs VdShocDecompandSolveData(10, 71, 72, 5, 19), VdShocDecompandSolveData(10, 12, 13, 2.5, 7), @@ -28,54 +28,54 @@ struct UnitWrap::UnitTest::TestVdShocDecompandSolve { VdShocDecompandSolveData(2, 7, 8, 1, 1) }; - static constexpr Int num_runs = sizeof(f90_data) / sizeof(VdShocDecompandSolveData); + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(VdShocDecompandSolveData); // Generate random input data. Diagonals in solver data will be overwritten // after results of decomp routine. for (Int i = 0; i < num_runs; ++i) { - VdShocDecompandSolveData& d_f90 = f90_data[i]; - d_f90.randomize(engine); + VdShocDecompandSolveData& d_baseline = baseline_data[i]; + d_baseline.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state VdShocDecompandSolveData cxx_data[] = { - VdShocDecompandSolveData(f90_data[0]), - VdShocDecompandSolveData(f90_data[1]), - VdShocDecompandSolveData(f90_data[2]), - VdShocDecompandSolveData(f90_data[3]) + VdShocDecompandSolveData(baseline_data[0]), + VdShocDecompandSolveData(baseline_data[1]), + VdShocDecompandSolveData(baseline_data[2]), + VdShocDecompandSolveData(baseline_data[3]) }; // Assume all data is in C layout - // Get data from fortran. - for (Int i = 0; i < num_runs; ++i) { - VdShocDecompandSolveData& d_f90 = f90_data[i]; - // expects data in C layout - vd_shoc_decomp_and_solve(d_f90); + // Read baseline data. + if (this->m_baseline_action == COMPARE) { + for (auto& d: baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (Int i = 0; i < num_runs; ++i) { VdShocDecompandSolveData& d_cxx = cxx_data[i]; - - d_cxx.transpose(); // _f expects data in fortran layout - vd_shoc_decomp_and_solve_f(d_cxx.shcol, d_cxx.nlev, d_cxx.nlevi, d_cxx.n_rhs, - d_cxx.kv_term, d_cxx.tmpi, d_cxx.rdp_zt, - d_cxx.dtime, d_cxx.flux, d_cxx.var); - d_cxx.transpose(); // go back to C layout + vd_shoc_decomp_and_solve(d_cxx); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - VdShocDecompandSolveData& d_f90 = f90_data[i]; + VdShocDecompandSolveData& d_baseline = baseline_data[i]; VdShocDecompandSolveData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.var); ++k) { - REQUIRE(d_f90.total(d_f90.var) == d_cxx.total(d_cxx.var)); - REQUIRE(d_f90.var[k] == d_cxx.var[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.var); ++k) { + REQUIRE(d_baseline.total(d_baseline.var) == d_cxx.total(d_cxx.var)); + REQUIRE(d_baseline.var[k] == d_cxx.var[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + cxx_data[i].write(Base::m_fid); + } } } // run_bfb @@ -91,7 +91,7 @@ TEST_CASE("vd_shoc_solve_bfb", "[shoc]") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestVdShocDecompandSolve; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_vertflux_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_vertflux_tests.cpp index 7ac7c7c53160..895040f9b11a 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_vertflux_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_vertflux_tests.cpp @@ -3,7 +3,7 @@ #include "shoc_unit_tests_common.hpp" #include "physics/share/physics_constants.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestCalcShocVertflux { +struct UnitWrap::UnitTest::TestCalcShocVertflux : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; static constexpr Int nlev = 4; @@ -96,10 +96,7 @@ struct UnitWrap::UnitTest::TestCalcShocVertflux { } // Call the C++ implementation - SDS.transpose(); - // expects data in fortran layout - calc_shoc_vertflux_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.tkh_zi, SDS.dz_zi, SDS.invar, SDS.vertflux); - SDS.transpose(); + calc_shoc_vertflux(SDS); // Check the results for(Int s = 0; s < shcol; ++s) { @@ -133,11 +130,11 @@ struct UnitWrap::UnitTest::TestCalcShocVertflux { } } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - CalcShocVertfluxData SDS_f90[] = { + CalcShocVertfluxData SDS_baseline[] = { // shcol, nlev, nlevi CalcShocVertfluxData(10, 71, 72), CalcShocVertfluxData(10, 12, 13), @@ -146,45 +143,49 @@ struct UnitWrap::UnitTest::TestCalcShocVertflux { }; // Generate random input data - for (auto& d : SDS_f90) { + for (auto& d : SDS_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state CalcShocVertfluxData SDS_cxx[] = { - CalcShocVertfluxData(SDS_f90[0]), - CalcShocVertfluxData(SDS_f90[1]), - CalcShocVertfluxData(SDS_f90[2]), - CalcShocVertfluxData(SDS_f90[3]), + CalcShocVertfluxData(SDS_baseline[0]), + CalcShocVertfluxData(SDS_baseline[1]), + CalcShocVertfluxData(SDS_baseline[2]), + CalcShocVertfluxData(SDS_baseline[3]), }; + static constexpr Int num_runs = sizeof(SDS_baseline) / sizeof(CalcShocVertfluxData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : SDS_f90) { - // expects data in C layout - calc_shoc_vertflux(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : SDS_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : SDS_cxx) { - d.transpose(); - // expects data in fortran layout - calc_shoc_vertflux_f(d.shcol, d.nlev, d.nlevi, d.tkh_zi, d.dz_zi, d.invar, d.vertflux); - d.transpose(); + calc_shoc_vertflux(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(SDS_f90) / sizeof(CalcShocVertfluxData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - CalcShocVertfluxData& d_f90 = SDS_f90[i]; + CalcShocVertfluxData& d_baseline = SDS_baseline[i]; CalcShocVertfluxData& d_cxx = SDS_cxx[i]; - for (Int k = 0; k < d_f90.total(d_f90.vertflux); ++k) { - REQUIRE(d_f90.vertflux[k] == d_cxx.vertflux[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.vertflux); ++k) { + REQUIRE(d_baseline.vertflux[k] == d_cxx.vertflux[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + SDS_cxx[i].write(Base::m_fid); + } } } @@ -200,14 +201,14 @@ TEST_CASE("shoc_vertflux_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestCalcShocVertflux; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_vertflux_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestCalcShocVertflux; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_w3_diag_third_moms_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_w3_diag_third_moms_tests.cpp deleted file mode 100644 index 39e9b63ba517..000000000000 --- a/components/eamxx/src/physics/shoc/tests/shoc_w3_diag_third_moms_tests.cpp +++ /dev/null @@ -1,122 +0,0 @@ -#include "catch2/catch.hpp" - -#include "shoc_unit_tests_common.hpp" -#include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" -#include "physics/share/physics_constants.hpp" -#include "share/scream_types.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/util/ekat_arch.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" - -#include -#include -#include -#include - -namespace scream { -namespace shoc { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestW3diagThirdMoms { - - static void run_property() - { - - // Tests for the SHOC function: - // w3_diag_third_shoc_moment - - // TEST ONE - // Do series of tests to make sure output is as expected - - // aa0 term - constexpr static Real aa0_test1 = -0.2; - // aa1 term - constexpr static Real aa1_test1 = 5.65; - // x0 term - constexpr static Real x0_test1 = -4.31; - // y0 term - constexpr static Real x1_test1 = 41.05; - // f5 term - constexpr static Real f5_test1 = 4; - - // Initialize data structure for bridging to F90 - W3DiagThirdShocMomentData SDS; - - // Load up the data - SDS.aa0 = aa0_test1; - SDS.aa1 = aa1_test1; - SDS.x0 = x0_test1; - SDS.x1 = x1_test1; - SDS.f5 = f5_test1; - - // Call the fortran implementation - w3_diag_third_shoc_moment(SDS); - - // Verify result is negative - REQUIRE(SDS.w3 < 0); - - // TEST TWO - // Modify parameters to decrease w3 - // decrease this term - constexpr static Real aa1_test2 = 2.65; - - SDS.aa1 = aa1_test2; - - // Call the fortran implementation - w3_diag_third_shoc_moment(SDS); - - // Verify result has decreased - REQUIRE(SDS.w3 < SDS.aa1); - - // TEST THREE - // Modify parameters to get positive result - // x0 term - constexpr static Real x0_test3 = -4.31; - // y0 term - constexpr static Real x1_test3 = -41.05; - // f5 term - constexpr static Real f5_test3 = -4; - - SDS.x0 = x0_test3; - SDS.x1 = x1_test3; - SDS.f5 = f5_test3; - - // Call the fortran implementation - w3_diag_third_shoc_moment(SDS); - - // Verify the result is positive - REQUIRE(SDS.w3 > 0); - - } - - static void run_bfb() - { - // TODO - } - -}; - -} // namespace unit_test -} // namespace shoc -} // namespace scream - -namespace{ - -TEST_CASE("shoc_w3_diag_third_moms_property", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestW3diagThirdMoms; - - TestStruct::run_property(); -} - -TEST_CASE("shoc_w3_diag_third_moms_bfb", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestW3diagThirdMoms; - - TestStruct::run_bfb(); -} - -} // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_xy_diag_third_moms_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_xy_diag_third_moms_tests.cpp deleted file mode 100644 index ad6e332e7316..000000000000 --- a/components/eamxx/src/physics/shoc/tests/shoc_xy_diag_third_moms_tests.cpp +++ /dev/null @@ -1,112 +0,0 @@ -#include "catch2/catch.hpp" - -#include "shoc_unit_tests_common.hpp" -#include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" -#include "physics/share/physics_constants.hpp" -#include "share/scream_types.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/util/ekat_arch.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" - -#include -#include -#include -#include - -namespace scream { -namespace shoc { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestXYdiagThirdMoms { - - static void run_property() - { - - // Tests for the SHOC function: - // x_y_terms_diag_third_shoc_moment - - // TEST ONE - // Call the function twice, with two different sets of terms. - // In the second test the fterms should be larger. Verify - // that the x0 and y0 terms are the same, but that the - // x1 and y1 terms have increased. - - // buoyancy term (isotropy squared * brunt vaisalla frequency) - constexpr static Real buoy_sgs2 = -100; - // f0 term - constexpr static Real f0_test1a = 13000; - // f3 term - constexpr static Real f1_test1a = 8500; - // f4 term - constexpr static Real f2_test1a = 30; - - // Initialize data structure for bridging to F90 - XYTermsDiagThirdShocMomentData SDS; - - // Load up the data - SDS.buoy_sgs2 = buoy_sgs2; - SDS.f0 = f0_test1a; - SDS.f1 = f1_test1a; - SDS.f2 = f2_test1a; - - // Call the fortran implementation - x_y_terms_diag_third_shoc_moment(SDS); - - // Save test results - Real x0_test1a = SDS.x0; - Real y0_test1a = SDS.y0; - Real x1_test1a = SDS.x1; - Real y1_test1a = SDS.y1; - - // Now load up data for second part of test - // Feed in LARGER values - SDS.f0 = 1.2*f0_test1a; - SDS.f1 = 1.2*f1_test1a; - SDS.f2 = 1.2*f2_test1a; - - // Call the fortran implementation - x_y_terms_diag_third_shoc_moment(SDS); - - // Now check the result - - // x0 and y0 terms should NOT have changed - REQUIRE(SDS.x0 == x0_test1a); - REQUIRE(SDS.y0 == y0_test1a); - - // x1 and y1 terms should have increased - REQUIRE(SDS.x1 > x1_test1a); - REQUIRE(SDS.y1 > y1_test1a); - - } - - static void run_bfb() - { - // TODO - } - -}; - -} // namespace unit_test -} // namespace shoc -} // namespace scream - -namespace{ - -TEST_CASE("shoc_xy_diag_third_moms_property", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestXYdiagThirdMoms; - - TestStruct::run_property(); -} - -TEST_CASE("shoc_xy_diag_third_moms_bfb", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestXYdiagThirdMoms; - - TestStruct::run_bfb(); -} - -} // namespace diff --git a/components/eamxx/src/physics/spa/eamxx_spa_process_interface.cpp b/components/eamxx/src/physics/spa/eamxx_spa_process_interface.cpp index 61c61b878c2f..c27af18f85b2 100644 --- a/components/eamxx/src/physics/spa/eamxx_spa_process_interface.cpp +++ b/components/eamxx/src/physics/spa/eamxx_spa_process_interface.cpp @@ -64,11 +64,11 @@ void SPA::set_grids(const std::shared_ptr grids_manager) // where a single column of data corresponding to the closest lat/lon pair to // the IOP lat/lon parameters is read from file, and that column data is mapped // to all columns of the IdentityRemapper source fields. - EKAT_REQUIRE_MSG(spa_map_file == "" or spa_map_file == "None" or not m_iop, + EKAT_REQUIRE_MSG(spa_map_file == "" or spa_map_file == "None" or not m_iop_data_manager, "Error! Cannot define spa_remap_file for cases with an Intensive Observation Period defined. " "The IOP class defines it's own remap from file data -> model data.\n"); - SPAHorizInterp = SPAFunc::create_horiz_remapper (m_grid,spa_data_file,spa_map_file, m_iop!=nullptr); + SPAHorizInterp = SPAFunc::create_horiz_remapper (m_grid,spa_data_file,spa_map_file, m_iop_data_manager!=nullptr); // Grab a sw and lw field from the horiz interp, and check sw/lw dim against what we hardcoded in this class auto nswbands_data = SPAHorizInterp->get_src_field(4).get_header().get_identifier().get_layout().dim("swband"); @@ -128,8 +128,8 @@ void SPA::set_grids(const std::shared_ptr grids_manager) // AtmosphereInput object (for reading into standard // grids) or a SpaFunctions::IOPReader (for reading into // an IOP grid). - if (m_iop) { - SPAIOPDataReader = SPAFunc::create_spa_data_reader(m_iop,SPAHorizInterp,spa_data_file); + if (m_iop_data_manager) { + SPAIOPDataReader = SPAFunc::create_spa_data_reader(m_iop_data_manager,SPAHorizInterp,spa_data_file); } else { SPADataReader = SPAFunc::create_spa_data_reader(SPAHorizInterp,spa_data_file); } diff --git a/components/eamxx/src/physics/spa/spa_functions.hpp b/components/eamxx/src/physics/spa/spa_functions.hpp index 6bdba702e70f..444b2d9992e8 100644 --- a/components/eamxx/src/physics/spa/spa_functions.hpp +++ b/components/eamxx/src/physics/spa/spa_functions.hpp @@ -4,7 +4,7 @@ #include "share/grid/abstract_grid.hpp" #include "share/grid/remap/abstract_remapper.hpp" #include "share/io/scorpio_input.hpp" -#include "share/iop/intensive_observation_period.hpp" +#include "share/atm_process/IOPDataManager.hpp" #include "share/util/scream_time_stamp.hpp" #include "share/scream_types.hpp" @@ -30,7 +30,7 @@ struct SPAFunctions using gid_type = AbstractGrid::gid_type; - using iop_ptr_type = std::shared_ptr; + using iop_data_ptr_type = std::shared_ptr; template using view_1d = typename KT::template view_1d; @@ -128,11 +128,11 @@ struct SPAFunctions }; // SPAInput struct IOPReader { - IOPReader (iop_ptr_type& iop_, + IOPReader (iop_data_ptr_type& iop_, const std::string file_name_, const std::vector& io_fields_, const std::shared_ptr& io_grid_) - : iop(iop_), file_name(file_name_) + : iop_data_manager(iop_), file_name(file_name_) { field_mgr = std::make_shared(io_grid_); for (auto& f : io_fields_) { @@ -141,14 +141,14 @@ struct SPAFunctions } // Set IO info for this grid and file in IOP object - iop->setup_io_info(file_name, io_grid_); + iop_data_manager->setup_io_info(file_name, io_grid_); } void read_variables(const int time_index, const util::TimeStamp& ts) { - iop->read_fields_from_file_for_iop(file_name, field_names, ts, field_mgr, time_index); + iop_data_manager->read_fields_from_file_for_iop(file_name, field_names, ts, field_mgr, time_index); } - iop_ptr_type iop; + iop_data_ptr_type iop_data_manager; std::string file_name; std::vector field_names; std::shared_ptr field_mgr; @@ -175,7 +175,7 @@ struct SPAFunctions static std::shared_ptr create_spa_data_reader ( - iop_ptr_type& iop, + iop_data_ptr_type& iop_data_manager, const std::shared_ptr& horiz_remapper, const std::string& spa_data_file); diff --git a/components/eamxx/src/physics/spa/spa_functions_impl.hpp b/components/eamxx/src/physics/spa/spa_functions_impl.hpp index 287f69a98896..24195b9da1aa 100644 --- a/components/eamxx/src/physics/spa/spa_functions_impl.hpp +++ b/components/eamxx/src/physics/spa/spa_functions_impl.hpp @@ -162,7 +162,7 @@ template std::shared_ptr::IOPReader> SPAFunctions:: create_spa_data_reader ( - iop_ptr_type& iop, + iop_data_ptr_type& iop_data_manager, const std::shared_ptr& horiz_remapper, const std::string& spa_data_file) { @@ -171,7 +171,7 @@ create_spa_data_reader ( io_fields.push_back(horiz_remapper->get_src_field(i)); } const auto io_grid = horiz_remapper->get_src_grid(); - return std::make_shared(iop, spa_data_file, io_fields, io_grid); + return std::make_shared(iop_data_manager, spa_data_file, io_fields, io_grid); } /*-----------------------------------------------------------------*/ @@ -521,14 +521,14 @@ ::update_spa_data_from_file( // Set the first/last entries of the spa data, so that linear interp // can extrapolate if the p_tgt is outside the p_src bounds Kokkos::single(Kokkos::PerTeam(team),[&]{ - spa_data_ccn3(icol,0) = 0; + spa_data_ccn3(icol,0) = ccn3(icol,0); for (int isw=0; isw("T_mid", scalar3d_mid, K, grid_name, ps); add_field("p_mid", scalar3d_mid, Pa, grid_name, ps); add_field("pseudo_density", scalar3d_mid, Pa, grid_name, ps); - add_field("qv", scalar3d_mid, kg/kg, grid_name, "tracers", ps); add_field("sgh30", scalar2d , m, grid_name); add_field("landfrac", scalar2d , nondim, grid_name); + add_tracer("qv", m_grid, kg/kg, ps); add_field("surf_drag_coeff_tms", scalar2d, kg/(m2*s), grid_name); add_field("wind_stress_tms", vector2d, N/m2, grid_name); diff --git a/components/eamxx/src/python/libpyscream/pyatmproc.hpp b/components/eamxx/src/python/libpyscream/pyatmproc.hpp index 219837c5f736..086b4467f996 100644 --- a/components/eamxx/src/python/libpyscream/pyatmproc.hpp +++ b/components/eamxx/src/python/libpyscream/pyatmproc.hpp @@ -183,7 +183,8 @@ struct PyAtmProc { // Create/setup the output mgr output_mgr = std::make_shared(); - output_mgr->setup(comm,params,fms,gm,t0,t0,false); + output_mgr->initialize(comm, params, t0, false); + output_mgr->setup(fms,gm); output_mgr->set_logger(ap->get_logger()); } diff --git a/components/eamxx/src/share/CMakeLists.txt b/components/eamxx/src/share/CMakeLists.txt index 105b39a98086..f0263e4a2253 100644 --- a/components/eamxx/src/share/CMakeLists.txt +++ b/components/eamxx/src/share/CMakeLists.txt @@ -8,6 +8,7 @@ set(SHARE_SRC atm_process/atmosphere_process_group.cpp atm_process/atmosphere_process_dag.cpp atm_process/atmosphere_diagnostic.cpp + atm_process/IOPDataManager.cpp field/field_alloc_prop.cpp field/field_identifier.cpp field/field_header.cpp @@ -27,18 +28,17 @@ set(SHARE_SRC grid/remap/horiz_interp_remapper_data.cpp grid/remap/refining_remapper_p2p.cpp grid/remap/vertical_remapper.cpp - iop/intensive_observation_period.cpp property_checks/property_check.cpp property_checks/field_nan_check.cpp property_checks/field_within_interval_check.cpp property_checks/mass_and_energy_column_conservation_check.cpp + util/eamxx_data_interpolation.cpp util/eamxx_fv_phys_rrtmgp_active_gases_workaround.cpp + util/eamxx_time_interpolation.cpp util/scream_time_stamp.cpp util/scream_timing.cpp util/scream_utils.cpp - util/eamxx_time_interpolation.cpp util/scream_bfbhash.cpp - util/eamxx_time_interpolation.cpp ) if (EAMXX_ENABLE_EXPERIMENTAL_CODE) @@ -49,15 +49,14 @@ if (EAMXX_ENABLE_EXPERIMENTAL_CODE) endif() add_library(scream_share ${SHARE_SRC}) -set_target_properties(scream_share PROPERTIES - Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/modules -) target_include_directories(scream_share PUBLIC ${SCREAM_SRC_DIR} ${SCREAM_BIN_DIR}/src ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_BINARY_DIR}/modules ) +# Used in the data interpolation +target_link_libraries(scream_share PUBLIC stdc++fs) + if (GPTL_PATH) target_include_directories(scream_share PUBLIC ${GPTL_PATH}) diff --git a/components/eamxx/src/share/iop/intensive_observation_period.cpp b/components/eamxx/src/share/atm_process/IOPDataManager.cpp similarity index 98% rename from components/eamxx/src/share/iop/intensive_observation_period.cpp rename to components/eamxx/src/share/atm_process/IOPDataManager.cpp index 32b120c57002..2e2c3552d8e4 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.cpp +++ b/components/eamxx/src/share/atm_process/IOPDataManager.cpp @@ -1,7 +1,7 @@ #include "share/grid/point_grid.hpp" #include "share/io/scorpio_input.hpp" #include "share/io/scream_scorpio_interface.hpp" -#include "share/iop/intensive_observation_period.hpp" +#include "share/atm_process/IOPDataManager.hpp" #include "ekat/ekat_assert.hpp" #include "ekat/util/ekat_lin_interp.hpp" @@ -27,13 +27,13 @@ namespace ekat { namespace scream { namespace control { -IntensiveObservationPeriod:: -IntensiveObservationPeriod(const ekat::Comm& comm, - const ekat::ParameterList& params, - const util::TimeStamp& run_t0, - const int model_nlevs, - const Field& hyam, - const Field& hybm) +IOPDataManager:: +IOPDataManager(const ekat::Comm& comm, + const ekat::ParameterList& params, + const util::TimeStamp& run_t0, + const int model_nlevs, + const Field& hyam, + const Field& hybm) { m_comm = comm; m_params = params; @@ -72,14 +72,14 @@ IntensiveObservationPeriod(const ekat::Comm& comm, initialize_iop_file(run_t0, model_nlevs); } -IntensiveObservationPeriod:: -~IntensiveObservationPeriod () +IOPDataManager:: +~IOPDataManager () { const auto iop_file = m_params.get("iop_file"); scorpio::release_file(iop_file); } -void IntensiveObservationPeriod:: +void IOPDataManager:: initialize_iop_file(const util::TimeStamp& run_t0, int model_nlevs) { @@ -270,7 +270,7 @@ initialize_iop_file(const util::TimeStamp& run_t0, scorpio::read_var(iop_file,"lon",&iop_file_lon); const Real rel_lat_err = std::fabs(iop_file_lat - m_params.get("target_latitude"))/ - std::max(m_params.get("target_latitude"),(Real)0.1); + std::max(std::fabs(m_params.get("target_latitude")),(Real)0.1); const Real rel_lon_err = std::fabs(std::fmod(iop_file_lon + 360.0, 360.0)-m_params.get("target_longitude"))/ std::max(m_params.get("target_longitude"),(Real)0.1); EKAT_REQUIRE_MSG(rel_lat_err < std::numeric_limits::epsilon(), @@ -310,7 +310,7 @@ initialize_iop_file(const util::TimeStamp& run_t0, m_helper_fields.insert({"model_pressure", model_pressure}); } -void IntensiveObservationPeriod:: +void IOPDataManager:: setup_io_info(const std::string& file_name, const grid_ptr& grid) { @@ -397,7 +397,7 @@ setup_io_info(const std::string& file_name, } } -void IntensiveObservationPeriod:: +void IOPDataManager:: read_fields_from_file_for_iop (const std::string& file_name, const vos& field_names_nc, const vos& field_names_eamxx, @@ -501,7 +501,7 @@ read_fields_from_file_for_iop (const std::string& file_name, } } -void IntensiveObservationPeriod:: +void IOPDataManager:: read_iop_file_data (const util::TimeStamp& current_ts) { // Query to see if we need to load data from IOP file. @@ -749,7 +749,7 @@ read_iop_file_data (const util::TimeStamp& current_ts) m_time_info.time_idx_of_current_data = iop_file_time_idx; } -void IntensiveObservationPeriod:: +void IOPDataManager:: set_fields_from_iop_data(const field_mgr_ptr field_mgr) { if (m_params.get("zero_non_iop_tracers") && field_mgr->has_group("tracers")) { @@ -858,7 +858,7 @@ set_fields_from_iop_data(const field_mgr_ptr field_mgr) }); } -void IntensiveObservationPeriod:: +void IOPDataManager:: correct_temperature_and_water_vapor(const field_mgr_ptr field_mgr) { // Find the first valid level index for t_iop, i.e., first non-zero entry diff --git a/components/eamxx/src/share/iop/intensive_observation_period.hpp b/components/eamxx/src/share/atm_process/IOPDataManager.hpp similarity index 92% rename from components/eamxx/src/share/iop/intensive_observation_period.hpp rename to components/eamxx/src/share/atm_process/IOPDataManager.hpp index 860eb082c427..0ca5e2ef6d10 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.hpp +++ b/components/eamxx/src/share/atm_process/IOPDataManager.hpp @@ -14,11 +14,9 @@ namespace scream { namespace control { /* - * Class which provides functionality for running EAMxx with an intensive - * observation period (IOP). Currently the only use case is the doubly - * periodic model (DP-SCREAM). + * Class which data for an intensive observation period (IOP). */ -class IntensiveObservationPeriod +class IOPDataManager { using vos = std::vector; using field_mgr_ptr = std::shared_ptr; @@ -47,15 +45,15 @@ class IntensiveObservationPeriod // - run_t0: Initial timestamp for the simulation // - model_nlevs: Number of vertical levels in the simulation. Needed since // the iop file contains a (potentially) different number of levels - IntensiveObservationPeriod(const ekat::Comm& comm, - const ekat::ParameterList& params, - const util::TimeStamp& run_t0, - const int model_nlevs, - const Field& hyam, - const Field& hybm); + IOPDataManager(const ekat::Comm& comm, + const ekat::ParameterList& params, + const util::TimeStamp& run_t0, + const int model_nlevs, + const Field& hyam, + const Field& hybm); - // Default destructor - ~IntensiveObservationPeriod(); + // Destructor + ~IOPDataManager(); // Read data from IOP file and store internally. void read_iop_file_data(const util::TimeStamp& current_ts); @@ -197,7 +195,7 @@ class IntensiveObservationPeriod std::map m_iop_file_varnames; std::map m_iop_field_surface_varnames; std::map m_iop_field_type; -}; // class IntensiveObservationPeriod +}; // class IOPDataManager } // namespace control } // namespace scream diff --git a/components/eamxx/src/share/atm_process/atmosphere_process.cpp b/components/eamxx/src/share/atm_process/atmosphere_process.cpp index ab6d34310ceb..6a85097539ae 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process.cpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process.cpp @@ -896,6 +896,8 @@ get_field_in_impl(const std::string& field_name, const std::string& grid_name) c " field name: " + field_name + "\n" " grid name: " + grid_name + "\n"); } + static Field f; + return f; } Field& AtmosphereProcess:: @@ -917,6 +919,8 @@ get_field_in_impl(const std::string& field_name) const { " atm proc name: " + this->name() + "\n" " field name: " + field_name + "\n"); } + static Field f; + return f; } Field& AtmosphereProcess:: @@ -932,6 +936,8 @@ get_field_out_impl(const std::string& field_name, const std::string& grid_name) " field name: " + field_name + "\n" " grid name: " + grid_name + "\n"); } + static Field f; + return f; } Field& AtmosphereProcess:: @@ -953,6 +959,8 @@ get_field_out_impl(const std::string& field_name) const { " atm proc name: " + this->name() + "\n" " field name: " + field_name + "\n"); } + static Field f; + return f; } FieldGroup& AtmosphereProcess:: @@ -968,6 +976,8 @@ get_group_in_impl(const std::string& group_name, const std::string& grid_name) c " group name: " + group_name + "\n" " grid name: " + grid_name + "\n"); } + static FieldGroup g(""); + return g; } FieldGroup& AtmosphereProcess:: @@ -989,6 +999,8 @@ get_group_in_impl(const std::string& group_name) const { " atm proc name: " + this->name() + "\n" " group name: " + group_name + "\n"); } + static FieldGroup g(""); + return g; } FieldGroup& AtmosphereProcess:: @@ -1004,6 +1016,8 @@ get_group_out_impl(const std::string& group_name, const std::string& grid_name) " group name: " + group_name + "\n" " grid name: " + grid_name + "\n"); } + static FieldGroup g(""); + return g; } FieldGroup& AtmosphereProcess:: @@ -1025,6 +1039,8 @@ get_group_out_impl(const std::string& group_name) const { " atm proc name: " + this->name() + "\n" " group name: " + group_name + "\n"); } + static FieldGroup g(""); + return g; } Field& AtmosphereProcess:: @@ -1040,6 +1056,8 @@ get_internal_field_impl(const std::string& field_name, const std::string& grid_n " field name: " + field_name + "\n" " grid name: " + grid_name + "\n"); } + static Field f; + return f; } Field& AtmosphereProcess:: @@ -1061,6 +1079,8 @@ get_internal_field_impl(const std::string& field_name) const { " atm proc name: " + this->name() + "\n" " field name: " + field_name + "\n"); } + static Field f; + return f; } void AtmosphereProcess diff --git a/components/eamxx/src/share/atm_process/atmosphere_process.hpp b/components/eamxx/src/share/atm_process/atmosphere_process.hpp index 3150f56c9f9a..016321ea506e 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process.hpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process.hpp @@ -1,10 +1,10 @@ #ifndef SCREAM_ATMOSPHERE_PROCESS_HPP #define SCREAM_ATMOSPHERE_PROCESS_HPP -#include "share/iop/intensive_observation_period.hpp" #include "share/atm_process/atmosphere_process_utils.hpp" #include "share/atm_process/ATMBufferManager.hpp" #include "share/atm_process/SCDataManager.hpp" +#include "share/atm_process/IOPDataManager.hpp" #include "share/field/field_identifier.hpp" #include "share/field/field_manager.hpp" #include "share/property_checks/property_check.hpp" @@ -83,7 +83,7 @@ class AtmosphereProcess : public ekat::enable_shared_from_this; - using iop_ptr = std::shared_ptr; + using iop_data_ptr = std::shared_ptr; // Base constructor to set MPI communicator and params AtmosphereProcess (const ekat::Comm& comm, const ekat::ParameterList& params); @@ -274,13 +274,14 @@ class AtmosphereProcess : public ekat::enable_shared_from_this get_logger () const { @@ -348,6 +349,12 @@ class AtmosphereProcess : public ekat::enable_shared_from_this& groups, const int ps) { add_field(FieldRequest(fid,groups,ps)); } + // Specialization for add_field to tracer group + template + void add_tracer (const std::string& name, std::shared_ptr grid, + const ekat::units::Units& u, const int ps = 1) + { add_field(name, grid->get_3d_scalar_layout(true), u, grid->name(), "tracers", ps); } + // Group requests template void add_group (const std::string& name, const std::string& grid, const int ps, const Bundling b, @@ -590,7 +597,7 @@ class AtmosphereProcess : public ekat::enable_shared_from_this\n" - << " " << html_fix(n.name) << ""; - if (verbosity>1) { + << " " << html_fix(n.name) + << "\n"; + + int sz_comp = n.computed.size(), sz_req = n.required.size(), + sz_grcomp = n.gr_computed.size(), sz_grreq = n.gr_required.size(); + int nfield = sz_comp + sz_req + sz_grcomp + sz_grreq; + if (verbosity > 1 && nfield > 0) { // FieldIntentifier prints bare min with verb 0. // DAG starts printing fids with verb 2, so fid verb is verb-2; int fid_verb = verbosity-2; - ofile << "
\n"; + ofile << "
\n"; + + if (sz_comp > 0) { + // Computed fields + if (n.id == id_begin) { + ofile << " " + << "Atm input fields from previous time step:\n"; + } else if (n.id == id_IC) { + ofile << " " + << "Initial Fields:\n"; + } else if (n.id != id_end) { + ofile << " " + << "Computed Fields:\n"; + } - // Computed fields - if (n.name=="Begin of atm time step") { - ofile << " Atm input fields from previous time step:\n"; - } else if (n.name!="End of atm time step"){ - ofile << " Computed Fields:\n"; - } - for (const auto& fid : n.computed) { - std::string fc = " "; - ofile << " " << fc << html_fix(print_fid(m_fids[fid],fid_verb)) << "\n"; + for (const auto& fid : n.computed) { + std::string fc = " "; + ofile << " " << fc + << html_fix(print_fid(m_fids[fid_out], fid_verb)) + << "\n"; + } } - // Required fields - if (n.name=="End of atm time step") { - ofile << " Atm output fields for next time step:\n"; - } else if (n.name!="Begin of atm time step") { - ofile << " Required Fields:\n"; - } - for (const auto& fid : n.required) { - std::string fc = " "; - ofile << " " << fc << html_fix(print_fid(m_fids[fid],fid_verb)); - if (ekat::contains(m_unmet_deps.at(n.id),fid)) { - ofile << " *** MISSING ***"; + if (sz_req > 0) { + // Required fields + if (n.id == id_end) { + ofile << " " + << "Atm output fields for next time step:\n"; + } else if (n.id != id_begin && n.id != id_IC) { + ofile << " " + << "Required Fields:\n"; + } + for (const auto& fid : n.required) { + std::string fc = " "; + ofile << " " << fc << html_fix(print_fid(m_fids[fid],fid_verb)); + if (ekat::contains(unmet, fid)) { + ofile << " *** MISSING ***"; + } else if (ekat::contains(unmet, -fid)) { + ofile << " (Init. Cond.)"; + has_IC_field = true; + } + ofile << "\n"; } - ofile << "\n"; } // Computed groups - if (n.gr_computed.size()>0) { - if (n.name=="Begin of atm time step") { - ofile << " Atm Input groups:\n"; - } else if (n.name!="End of atm time step"){ - ofile << " Computed Groups:\n"; + if (sz_grcomp > 0) { + if (n.id == id_begin) { + ofile << " Atm Input groups:\n"; + } else if (n.id != id_end){ + ofile << " Computed Groups:\n"; } for (const auto& gr_fid : n.gr_computed) { std::string fc = " "; ofile << " " << fc << html_fix(print_fid(m_fids[gr_fid],fid_verb)); ofile << "\n"; @@ -242,10 +293,8 @@ void AtmProcDAG::write_dag (const std::string& fname, const int verbosity) const size_t i = 0; for (const auto& fn : members_names) { const auto f = members.at(fn); - const auto& mfid = f->get_header().get_identifier(); - const auto mfid_id = get_fid_index(mfid); std::string mfc = ""; if (len>0) { ofile << ","; @@ -269,18 +318,18 @@ void AtmProcDAG::write_dag (const std::string& fname, const int verbosity) const } // Required groups - if (n.gr_required.size()>0) { + if (sz_grreq > 0) { if (n.name=="End of atm time step") { - ofile << " Atm Output Groups:\n"; + ofile << " Atm Output Groups:\n"; } else if (n.name!="Begin of atm time step") { - ofile << " Required Groups:\n"; + ofile << " Required Groups:\n"; } for (const auto& gr_fid : n.gr_required) { std::string fc = " "; ofile << " " << fc << html_fix(print_fid(m_fids[gr_fid],fid_verb)); - if (ekat::contains(m_unmet_deps.at(n.id),gr_fid)) { + if (ekat::contains(unmet, gr_fid)) { ofile << " *** MISSING ***"; } ofile << "\n"; @@ -327,10 +376,45 @@ void AtmProcDAG::write_dag (const std::string& fname, const int verbosity) const // Write all outgoing edges for (const auto c : n.children) { - ofile << n.id << "->" << c << "\n"; + ofile << n.id << "->" << c << "[penwidth=4];\n"; } } + if (!m_IC_processed && m_has_unmet_deps) { + int this_node_id = m_nodes.size() + 1; + ofile << this_node_id << " [\n" + << " shape=box\n" + << " color=\"#605d57\"\n" + << " fontcolor=\"#034a4a\"\n" + << " penwidth=8\n" + << " fontsize=40\n" + << " style=filled\n" + << " fillcolor=\"#999999\"\n" + << " align=\"center\"\n" + << " label=<NOTE: " + "Fields marked missing may be
provided by " + "the as-yet-unprocessed
initial condition
>\n" + << "];\n"; + } + + if (m_IC_processed && has_IC_field) { + int this_node_id = m_nodes.size() + 1; + ofile << this_node_id << " [\n" + << " shape=box\n" + << " color=\"#605d57\"\n" + << " fontcolor=\"#031576\"\n" + << " penwidth=8\n" + << " fontsize=40\n" + << " style=filled\n" + << " fillcolor=\"#cccccc\"\n" + << " align=\"center\"\n" + << " label=<NOTE: Fields denoted " + "with green text " + "
indicate the field was provided by the " + "
initial conditions and never updated
>\n" + << "];\n"; + } + // Close the file ofile << "}"; ofile.close(); @@ -341,6 +425,7 @@ void AtmProcDAG::cleanup () { m_fid_to_last_provider.clear(); m_unmet_deps.clear(); m_has_unmet_deps = false; + m_IC_processed = false; } void AtmProcDAG:: @@ -364,10 +449,9 @@ add_nodes (const group_type& atm_procs) add_nodes(*group); } else { // Create a node for the process - // Node& node = m_nodes[proc->name()]; int id = m_nodes.size(); m_nodes.push_back(Node()); - Node& node = m_nodes.back();; + Node& node = m_nodes.back(); node.id = id; node.name = proc->name(); m_unmet_deps[id].clear(); // Ensures an entry for this id is in the map @@ -507,6 +591,69 @@ void AtmProcDAG::add_edges () { } } +void AtmProcDAG::process_initial_conditions(const grid_field_map &ic_inited) { + // First, add the fields that were determined to come from the previous time + // step => IC for t = 0 + + // Create a node for the ICs by copying the begin_node. Recall that so far + // m_nodes contains [, 'beg-of-step', 'end-of-step'] + // WARNING: do NOT get a ref to beg-of-step node, since calls to m_nodes.push_back + // may resize the vector, invalidating the reference. + auto begin_node = m_nodes[m_nodes.size()-2]; + auto& ic_node = m_nodes.emplace_back(begin_node); + + // now set/clear the basic data for the ic_node + int id = m_nodes.size(); + ic_node.id = id; + ic_node.name = "Initial Conditions"; + m_unmet_deps[id].clear(); + ic_node.children.clear(); + // now add the begin_node as a child of the ic_node + ic_node.children.push_back(begin_node.id); + // return if there's nothing to process in the ic_inited vector + if (ic_inited.size() == 0) { + return; + } + std::set to_be_marked; + for (auto &node : m_nodes) { + if (m_unmet_deps.at(node.id).empty()) { + continue; + } else { + // NOTE: node_unmet_fields is a std::set + auto &node_unmet_fields = m_unmet_deps.at(node.id); + // add the current node as a child of the IC node + ic_node.children.push_back(node.id); + for (auto &um_fid : node_unmet_fields) { + for (auto &it1 : ic_inited) { + const auto &grid_name = it1.first; + // if this unmet-dependency field's name is in the ic_inited map for + // the provided grid_name key, we record the field id in to_be_marked + // (because changing it messes up the iterator) + if (ekat::contains(ic_inited.at(grid_name), m_fids[um_fid].name())) { + to_be_marked.insert(um_fid); + // add the fid of the formerly unmet dep to the initial condition + // node's computed list + ic_node.computed.insert(um_fid); + } else { + continue; + } + } + } + if (to_be_marked.empty()) { + continue; + } else { + // change the previously unmet dependency's field id to be negative, + // indicating that it is now met and provided by the initial condition + for (auto &fid : to_be_marked) { + node_unmet_fields.erase(fid); + node_unmet_fields.insert(-fid); + } + } + } + } + m_IC_processed = true; +} + int AtmProcDAG::add_fid (const FieldIdentifier& fid) { auto it = ekat::find(m_fids,fid); if (it==m_fids.end()) { diff --git a/components/eamxx/src/share/atm_process/atmosphere_process_dag.hpp b/components/eamxx/src/share/atm_process/atmosphere_process_dag.hpp index 1a88381ecc8f..52a0f9d8cd96 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process_dag.hpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process_dag.hpp @@ -16,6 +16,11 @@ class AtmProcDAG { void create_dag (const group_type& atm_procs); + using grid_field_map = std::map>; + void process_initial_conditions(const grid_field_map &ic_inited); + + void init_atm_proc_nodes(const group_type& atm_procs); + void add_surface_coupling (const std::set& imports, const std::set& exports); @@ -66,6 +71,7 @@ class AtmProcDAG { // Map a node id to a set of unmet field dependencies std::map> m_unmet_deps; bool m_has_unmet_deps; + bool m_IC_processed; // The nodes in the atm DAG std::vector m_nodes; diff --git a/components/eamxx/src/share/atm_process/atmosphere_process_group.hpp b/components/eamxx/src/share/atm_process/atmosphere_process_group.hpp index 6af2b2434b35..43b5f576984b 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process_group.hpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process_group.hpp @@ -106,9 +106,9 @@ class AtmosphereProcessGroup : public AtmosphereProcess void add_additional_data_fields_to_property_checks (const Field& data_field); // Loop through all proceeses in group and set IOP object - void set_iop(const iop_ptr& iop) { + void set_iop_data_manager(const iop_data_ptr& iop_data_manager) { for (auto& atm_proc : m_atm_processes) { - atm_proc->set_iop(iop); + atm_proc->set_iop_data_manager(iop_data_manager); } } diff --git a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp index 533090e4e7cb..cb9d9690db97 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp @@ -113,19 +113,32 @@ void hash (const std::list& fgs, HashType& accum) { } // namespace anon +// (mem, nmem) describe an arbitrary device array. If non-0, the array will be +// hashed and reported as a fourth line. void AtmosphereProcess ::print_global_state_hash (const std::string& label, const bool in, const bool out, - const bool internal) const { - static constexpr int nslot = 3; + const bool internal, const Real* mem, const int nmem) const { + static constexpr int nslot = 4; HashType laccum[nslot] = {0}; hash(m_fields_in, laccum[0]); hash(m_groups_in, laccum[0]); hash(m_fields_out, laccum[1]); hash(m_groups_out, laccum[1]); hash(m_internal_fields, laccum[2]); + const bool hash_array = mem != nullptr; + if (hash_array) { + HashType accum = 0; + Kokkos::parallel_reduce( + Kokkos::RangePolicy(0, nmem), + KOKKOS_LAMBDA(const int i, HashType& accum) { bfbhash::hash(mem[i], accum); }, + bfbhash::HashReducer<>(accum)); + Kokkos::fence(); + laccum[3] = accum; + } HashType gaccum[nslot]; - bfbhash::all_reduce_HashType(m_comm.mpi_comm(), laccum, gaccum, nslot); - const bool show[] = {in, out, internal}; + const int nr = hash_array ? nslot : nslot-1; + bfbhash::all_reduce_HashType(m_comm.mpi_comm(), laccum, gaccum, nr); + const bool show[] = {in, out, internal, hash_array}; if (m_comm.am_i_root()) for (int i = 0; i < nslot; ++i) if (show[i]) diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index a338199f2ef0..3fd15aa03ac6 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -349,6 +349,7 @@ class Field { get_subview_1 (const get_view_type,HD>&, const int) const { EKAT_ERROR_MSG ("Error! Cannot subview a rank2 view along the second " "dimension without losing LayoutRight.\n"); + return get_view_type,HD>(); } template diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index adbe36538c8a..e1d8c182d0e3 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -398,7 +398,7 @@ deep_copy_impl (const Field& src) const { if (rank == 0) { auto v = get_view< ST,HD>(); auto v_src = src.get_view(); - v() = v_src(); + Kokkos::deep_copy(v,v_src); return; } @@ -543,7 +543,7 @@ void Field::deep_copy_impl (const ST value) const { case 0: { auto v = get_view(); - v() = value; + Kokkos::deep_copy(v,value); } break; case 1: @@ -966,6 +966,7 @@ auto Field::get_ND_view () const "MaxRank = 6.\n" "This should never be called at run time.\n" "Please contact developer if this functionality is required\n"); + return get_view_type,HD>(); } } // namespace scream diff --git a/components/eamxx/src/share/field/field_utils.hpp b/components/eamxx/src/share/field/field_utils.hpp index 8f977a7caa1b..c6acbfc4079a 100644 --- a/components/eamxx/src/share/field/field_utils.hpp +++ b/components/eamxx/src/share/field/field_utils.hpp @@ -11,16 +11,18 @@ inline bool views_are_equal(const Field& f1, const Field& f2, const ekat::Comm* EKAT_REQUIRE_MSG (f1.data_type()==f2.data_type(), "Error! Views have different data types.\n"); + bool ret = false; switch (f1.data_type()) { case DataType::IntType: - return impl::views_are_equal(f1,f2,comm); + ret = impl::views_are_equal(f1,f2,comm); break; case DataType::FloatType: - return impl::views_are_equal(f1,f2,comm); + ret = impl::views_are_equal(f1,f2,comm); break; case DataType::DoubleType: - return impl::views_are_equal(f1,f2,comm); + ret = impl::views_are_equal(f1,f2,comm); break; default: EKAT_ERROR_MSG ("Error! Unrecognized field data type.\n"); } + return ret; } template @@ -111,6 +113,162 @@ void perturb (const Field& f, impl::perturb(f, engine, pdf, base_seed, level_mask, dof_gids); } +// Utility to compute the contraction of a field along its column dimension. +// This is equivalent to f_out = einsum('i,i...k->...k', weight, f_in). +// The impl is such that: +// - f_out, f_in, and weight must be provided and allocated +// - The first dimension is for the columns (COL) +// - There can be only up to 3 dimensions of f_in +template +void horiz_contraction(const Field &f_out, const Field &f_in, + const Field &weight, const ekat::Comm *comm = nullptr) { + using namespace ShortFieldTagsNames; + + const auto &l_out = f_out.get_header().get_identifier().get_layout(); + + const auto &l_in = f_in.get_header().get_identifier().get_layout(); + + const auto &l_w = weight.get_header().get_identifier().get_layout(); + + // Sanity checks before handing off to the implementation + EKAT_REQUIRE_MSG(l_w.rank() == 1, + "Error! The weight field must be rank-1.\n" + "The input has rank " + << l_w.rank() << ".\n"); + EKAT_REQUIRE_MSG(l_w.tags() == std::vector({COL}), + "Error! The weight field must have a column dimension.\n" + "The input f1 layout is " + << l_w.tags() << ".\n"); + EKAT_REQUIRE_MSG(l_in.rank() <= 3, + "Error! The input field must be at most rank-3.\n" + "The input f_in rank is " + << l_in.rank() << ".\n"); + EKAT_REQUIRE_MSG(l_in.tags()[0] == COL, + "Error! The input field must have a column dimension.\n" + "The input f_in layout is " + << l_in.to_string() << ".\n"); + EKAT_REQUIRE_MSG( + l_w.dim(0) == l_in.dim(0), + "Error! input and weight fields must have the same dimension along " + "which we are taking the reducing the field.\n" + "The weight field has dimension " + << l_w.dim(0) + << " while " + "the input field has dimension " + << l_in.dim(0) << ".\n"); + EKAT_REQUIRE_MSG( + l_in.dim(0) > 0, + "Error! The input field must have a non-zero column dimension.\n" + "The input f_in layout is " + << l_in.to_string() << ".\n"); + EKAT_REQUIRE_MSG( + l_out == l_in.clone().strip_dim(0), + "Error! The output field must have the same layout as the input field " + "without the column dimension.\n" + "The input f_in layout is " + << l_in.to_string() << " and the output f_out layout is " + << l_out.to_string() << ".\n"); + EKAT_REQUIRE_MSG( + f_out.is_allocated() && f_in.is_allocated() && weight.is_allocated(), + "Error! All fields must be allocated."); + EKAT_REQUIRE_MSG(f_out.data_type() == f_in.data_type(), + "Error! In/out Fields have matching data types."); + EKAT_REQUIRE_MSG( + f_out.data_type() == weight.data_type(), + "Error! Weight field must have the same data type as input fields."); + + // All good, call the implementation + impl::horiz_contraction(f_out, f_in, weight, comm); +} + +// Utility to compute the contraction of a field along its level dimension. +// This is equivalent to f_out = einsum('...k->...', weight, f_in). +// The impl is such that: +// - f_out, f_in, and weight must be provided and allocated +// - The last dimension is for the levels (LEV/ILEV) +// - There can be only up to 3 dimensions of f_in +// - Weight is assumed to be (in order of checking/impl): +// - rank-1, with only LEV/ILEV dimension +// - rank-2, with only COL and LEV/ILEV dimensions +template +void vert_contraction(const Field &f_out, const Field &f_in, + const Field &weight, const ekat::Comm *comm = nullptr) { + using namespace ShortFieldTagsNames; + + const auto &l_out = f_out.get_header().get_identifier().get_layout(); + + const auto &l_in = f_in.get_header().get_identifier().get_layout(); + + const auto &l_w = weight.get_header().get_identifier().get_layout(); + + // Sanity checks before handing off to the implementation + EKAT_REQUIRE_MSG( + l_w.rank() == 1 or l_w.rank() == 2, + "Error! The weight field must be at least rank-1 and at most rank-2.\n" + "The weight field has rank " + << l_w.rank() << ".\n"); + EKAT_REQUIRE_MSG( + l_w.tags().back() == LEV or l_w.tags().back() == ILEV, + "Error! The weight field must have LEV as its last dimension.\n" + "The weight field layout is " + << l_w.to_string() << ".\n"); + EKAT_REQUIRE_MSG(l_in.rank() <= 3, + "Error! The input field must be at most rank-3.\n" + "The input field rank is " + << l_in.rank() << ".\n"); + EKAT_REQUIRE_MSG(l_in.rank() >= l_w.rank(), + "Error! The input field must have at least as many " + "dimensions as the weight field.\n" + "The input field rank is " + << l_in.rank() << " and the weight field rank is " + << l_w.rank() << ".\n"); + EKAT_REQUIRE_MSG(l_in.tags().back() == LEV or l_in.tags().back() == ILEV, + "Error! The input field must have a level dimension.\n" + "The input field layout is " + << l_in.to_string() << ".\n"); + EKAT_REQUIRE_MSG( + l_in.dims().back() == l_w.dims().back(), + "Error! input and weight fields must have the same dimension along " + "which we are taking the reducing the field (last dimensions).\n" + "The weight field has last dimension " + << l_w.dims().back() + << " while " + "the input field has last dimension " + << l_in.dims().back() << ".\n"); + EKAT_REQUIRE_MSG( + l_in.dims().back() > 0, + "Error! The input field must have a non-zero level dimension.\n" + "The input field layout is " + << l_in.to_string() << ".\n"); + if(l_w.rank() == 2) { + EKAT_REQUIRE_MSG(l_w.congruent(l_in.clone().strip_dim(CMP, false)), + "Error! Incompatible layouts\n" + " field in: " + + l_in.to_string() + + "\n" + " weight: " + + l_w.to_string() + "\n"); + } + EKAT_REQUIRE_MSG( + l_out == l_in.clone().strip_dim(l_in.rank() - 1), + "Error! The output field must have the same layout as the input field " + "without the level dimension.\n" + "The input field layout is " + << l_in.to_string() << " and the output field layout is " + << l_out.to_string() << ".\n"); + EKAT_REQUIRE_MSG( + f_out.is_allocated() && f_in.is_allocated() && weight.is_allocated(), + "Error! All fields must be allocated."); + EKAT_REQUIRE_MSG(f_out.data_type() == f_in.data_type(), + "Error! In/out Fields have matching data types."); + EKAT_REQUIRE_MSG( + f_out.data_type() == weight.data_type(), + "Error! Weight field must have the same data type as input field."); + + // All good, call the implementation + impl::vert_contraction(f_out, f_in, weight, comm); +} + template ST frobenius_norm(const Field& f, const ekat::Comm* comm = nullptr) { diff --git a/components/eamxx/src/share/field/field_utils_impl.hpp b/components/eamxx/src/share/field/field_utils_impl.hpp index b771271852cf..94584cbd0432 100644 --- a/components/eamxx/src/share/field/field_utils_impl.hpp +++ b/components/eamxx/src/share/field/field_utils_impl.hpp @@ -5,6 +5,8 @@ #include "ekat/mpi/ekat_comm.hpp" +#include "ekat/kokkos/ekat_kokkos_utils.hpp" + #include #include @@ -36,6 +38,16 @@ bool views_are_equal(const Field& f1, const Field& f2, const ekat::Comm* comm) bool same_locally = true; const auto& dims = l1.dims(); switch (l1.rank()) { + case 0: + { + auto v1 = f1.template get_strided_view(); + auto v2 = f2.template get_strided_view(); + if (v1() != v2()) { + same_locally = false; + break; + } + break; + } case 1: { auto v1 = f1.template get_strided_view(); @@ -283,6 +295,165 @@ void perturb (const Field& f, } } +template +void horiz_contraction(const Field &f_out, const Field &f_in, + const Field &weight, const ekat::Comm *comm) { + using KT = ekat::KokkosTypes; + using RangePolicy = Kokkos::RangePolicy; + using TeamPolicy = Kokkos::TeamPolicy; + using TeamMember = typename TeamPolicy::member_type; + using ESU = ekat::ExeSpaceUtils; + + auto l_out = f_out.get_header().get_identifier().get_layout(); + auto l_in = f_in.get_header().get_identifier().get_layout(); + + auto v_w = weight.get_view(); + + const int ncols = l_in.dim(0); + + switch(l_in.rank()) { + case 1: { + auto v_in = f_in.get_view(); + auto v_out = f_out.get_view(); + Kokkos::parallel_reduce( + f_out.name(), RangePolicy(0, ncols), + KOKKOS_LAMBDA(const int i, ST &ls) { ls += v_w(i) * v_in(i); }, + v_out); + } break; + case 2: { + auto v_in = f_in.get_view(); + auto v_out = f_out.get_view(); + const int d1 = l_in.dim(1); + auto p = ESU::get_default_team_policy(d1, ncols); + Kokkos::parallel_for( + f_out.name(), p, KOKKOS_LAMBDA(const TeamMember &tm) { + const int j = tm.league_rank(); + Kokkos::parallel_reduce( + Kokkos::TeamVectorRange(tm, ncols), + [&](int i, ST &ac) { ac += v_w(i) * v_in(i, j); }, v_out(j)); + }); + } break; + case 3: { + auto v_in = f_in.get_view(); + auto v_out = f_out.get_view(); + const int d1 = l_in.dim(1); + const int d2 = l_in.dim(2); + auto p = ESU::get_default_team_policy(d1 * d2, ncols); + Kokkos::parallel_for( + f_out.name(), p, KOKKOS_LAMBDA(const TeamMember &tm) { + const int idx = tm.league_rank(); + const int j = idx / d2; + const int k = idx % d2; + Kokkos::parallel_reduce( + Kokkos::TeamVectorRange(tm, ncols), + [&](int i, ST &ac) { ac += v_w(i) * v_in(i, j, k); }, + v_out(j, k)); + }); + } break; + default: + EKAT_ERROR_MSG("Error! Unsupported field rank.\n"); + } + + if(comm) { + // TODO: use device-side MPI calls + // TODO: the dev ptr causes problems; revisit this later + // TODO: doing cuda-aware MPI allreduce would be ~10% faster + Kokkos::fence(); + f_out.sync_to_host(); + comm->all_reduce(f_out.template get_internal_view_data(), + l_out.size(), MPI_SUM); + f_out.sync_to_dev(); + } +} + +template +void vert_contraction(const Field &f_out, const Field &f_in, + const Field &weight, const ekat::Comm *comm) { + using KT = ekat::KokkosTypes; + using RangePolicy = Kokkos::RangePolicy; + using TeamPolicy = Kokkos::TeamPolicy; + using TeamMember = typename TeamPolicy::member_type; + using ESU = ekat::ExeSpaceUtils; + + auto l_out = f_out.get_header().get_identifier().get_layout(); + auto l_in = f_in.get_header().get_identifier().get_layout(); + auto l_w = weight.get_header().get_identifier().get_layout(); + + const int nlevs = l_in.dim(l_in.rank() - 1); + + // To avoid duplicating code for the 1d and 2d weight cases, + // we use a view to access the weight ahead of time + typename Field::get_view_type w1d; + typename Field::get_view_type w2d; + auto w_is_1d = l_w.rank() == 1; + if(w_is_1d) { + w1d = weight.get_view(); + } else { + w2d = weight.get_view(); + } + + switch(l_in.rank()) { + case 1: { + auto v_w = weight.get_view(); + auto v_in = f_in.get_view(); + auto v_out = f_out.get_view(); + Kokkos::parallel_reduce( + f_out.name(), RangePolicy(0, nlevs), + KOKKOS_LAMBDA(const int i, ST &ls) { ls += v_w(i) * v_in(i); }, + v_out); + } break; + case 2: { + auto v_in = f_in.get_view(); + auto v_out = f_out.get_view(); + const int d0 = l_in.dim(0); + auto p = ESU::get_default_team_policy(d0, nlevs); + Kokkos::parallel_for( + f_out.name(), p, KOKKOS_LAMBDA(const TeamMember &tm) { + const int i = tm.league_rank(); + Kokkos::parallel_reduce( + Kokkos::TeamVectorRange(tm, nlevs), + [&](int j, ST &ac) { + ac += w_is_1d ? w1d(j) * v_in(i, j) : w2d(i, j) * v_in(i, j); + }, + v_out(i)); + }); + } break; + case 3: { + auto v_in = f_in.get_view(); + auto v_out = f_out.get_view(); + const int d0 = l_in.dim(0); + const int d1 = l_in.dim(1); + auto p = ESU::get_default_team_policy(d0 * d1, nlevs); + Kokkos::parallel_for( + f_out.name(), p, KOKKOS_LAMBDA(const TeamMember &tm) { + const int idx = tm.league_rank(); + const int i = idx / d1; + const int j = idx % d1; + Kokkos::parallel_reduce( + Kokkos::TeamVectorRange(tm, nlevs), + [&](int k, ST &ac) { + ac += w_is_1d ? w1d(k) * v_in(i, j, k) + : w2d(i, k) * v_in(i, j, k); + }, + v_out(i, j)); + }); + } break; + default: + EKAT_ERROR_MSG("Error! Unsupported field rank in vert_contraction.\n"); + } + + if(comm) { + // TODO: use device-side MPI calls + // TODO: the dev ptr causes problems; revisit this later + // TODO: doing cuda-aware MPI allreduce would be ~10% faster + Kokkos::fence(); + f_out.sync_to_host(); + comm->all_reduce(f_out.template get_internal_view_data(), + l_out.size(), MPI_SUM); + f_out.sync_to_dev(); + } +} + template ST frobenius_norm(const Field& f, const ekat::Comm* comm) { diff --git a/components/eamxx/src/share/grid/abstract_grid.hpp b/components/eamxx/src/share/grid/abstract_grid.hpp index d88e05b29f02..c8917e3e287e 100644 --- a/components/eamxx/src/share/grid/abstract_grid.hpp +++ b/components/eamxx/src/share/grid/abstract_grid.hpp @@ -91,7 +91,6 @@ class AbstractGrid : public ekat::enable_shared_from_this FieldLayout get_2d_vector_layout (const int vector_dim) const; FieldLayout get_2d_tensor_layout (const std::vector& cmp_dims) const; - FieldLayout get_3d_vector_layout (const bool midpoints) const; FieldLayout get_3d_vector_layout (const bool midpoints, const int vector_dim) const; FieldLayout get_3d_tensor_layout (const bool midpoints, const std::vector& cmp_dims) const; diff --git a/components/eamxx/src/share/grid/grids_manager.cpp b/components/eamxx/src/share/grid/grids_manager.cpp index 77300aa51c90..45494947acad 100644 --- a/components/eamxx/src/share/grid/grids_manager.cpp +++ b/components/eamxx/src/share/grid/grids_manager.cpp @@ -7,7 +7,24 @@ auto GridsManager:: get_grid(const std::string& name) const -> grid_ptr_type { - auto g = get_grid_nonconst(name); + EKAT_REQUIRE_MSG (has_grid(name), + "Error! Grids manager '" + this->name() + "' does not provide grid '" + name + "'.\n" + " Avaialble grids are: " + print_available_grids() + "\n"); + + grid_ptr_type g; + for (const auto& it : m_grids) { + if (it.second->name()==name or + ekat::contains(it.second->aliases(),name)) { + g = it.second; + break; + } + } + + EKAT_REQUIRE_MSG (g!=nullptr, + "Something went wrong while looking up a grid.\n" + " - grids manager: " + this->name() + "\n" + " - grid name : " + name + "\n"); + return g; } @@ -51,7 +68,14 @@ create_remapper (const grid_ptr_type& from_grid, } void GridsManager:: -add_grid (nonconstgrid_ptr_type grid) +add_nonconst_grid (nonconstgrid_ptr_type grid) +{ + add_grid(grid); + m_nonconst_grids[grid->name()] = grid; +} + +void GridsManager:: +add_grid (grid_ptr_type grid) { const auto& name = grid->name(); EKAT_REQUIRE_MSG (not has_grid(name), @@ -59,7 +83,6 @@ add_grid (nonconstgrid_ptr_type grid) " - grids manager: " + this->name() + "\n" " - grid name : " + name + "\n"); - m_nonconst_grids[name] = grid; m_grids[name] = grid; } @@ -86,7 +109,6 @@ get_grid_nonconst (const std::string& name) const " - grid name : " + name + "\n"); return g; - } void GridsManager:: diff --git a/components/eamxx/src/share/grid/grids_manager.hpp b/components/eamxx/src/share/grid/grids_manager.hpp index 4a0e25bc233c..e70bb9678f01 100644 --- a/components/eamxx/src/share/grid/grids_manager.hpp +++ b/components/eamxx/src/share/grid/grids_manager.hpp @@ -56,7 +56,8 @@ class GridsManager protected: - void add_grid (nonconstgrid_ptr_type grid); + void add_nonconst_grid (nonconstgrid_ptr_type grid); + void add_grid (grid_ptr_type grid); void alias_grid (const std::string& grid_name, const std::string& grid_alias); virtual remapper_ptr_type diff --git a/components/eamxx/src/share/grid/library_grids_manager.hpp b/components/eamxx/src/share/grid/library_grids_manager.hpp new file mode 100644 index 000000000000..3e25ea603810 --- /dev/null +++ b/components/eamxx/src/share/grid/library_grids_manager.hpp @@ -0,0 +1,50 @@ +#ifndef EAMXX_LIBRARY_GRIDS_MANAGER_HPP +#define EAMXX_LIBRARY_GRIDS_MANAGER_HPP + +#include "share/grid/grids_manager.hpp" + +namespace scream { + +// This class is meant to be used within scopes that need a grids manager +// object, but they only have pre-built grids. The user can then simply +// create a LibraryGridsManager, and add the pre-existing grids. Afterwards, +// it's business as usual with GridsManager's interfaces. + +class LibraryGridsManager : public GridsManager +{ +public: + template + explicit LibraryGridsManager(Pointers&&... ptrs) { + add_grids(std::forward(ptrs)...); + } + + virtual ~LibraryGridsManager () = default; + + std::string name () const { return "Library grids_manager"; } + + void build_grids () override {} + + void add_grids () {} + + template + void add_grids (grid_ptr_type p, Pointers&&... ptrs) { + add_grid(p); + add_grids(std::forward(ptrs)...); + } + +protected: + remapper_ptr_type + do_create_remapper (const grid_ptr_type from_grid, + const grid_ptr_type to_grid) const + { + EKAT_ERROR_MSG ( + "Error! LibraryGridsManager is not capable of creating remappers.\n" + " - from_grid: " + from_grid->name() + "\n" + " - to_grid: " + to_grid->name() + "\n"); + return nullptr; + } +}; + +} // namespace scream + +#endif // EAMXX_LIBRARY_GRIDS_MANAGER_HPP diff --git a/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp b/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp index c930013f8398..4d505803e66f 100644 --- a/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp +++ b/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp @@ -106,7 +106,7 @@ build_se_grid (const std::string& name, ekat::ParameterList& params) se_grid->m_short_name = "se"; add_geo_data(se_grid); - add_grid(se_grid); + add_nonconst_grid(se_grid); } void MeshFreeGridsManager:: @@ -132,7 +132,7 @@ build_point_grid (const std::string& name, ekat::ParameterList& params) add_geo_data(pt_grid); pt_grid->m_short_name = "pt"; - add_grid(pt_grid); + add_nonconst_grid(pt_grid); } void MeshFreeGridsManager:: @@ -147,24 +147,30 @@ add_geo_data (const nonconstgrid_ptr_type& grid) const if (geo_data_source=="CREATE_EMPTY_DATA") { using namespace ShortFieldTagsNames; FieldLayout layout_mid ({LEV},{grid->get_num_vertical_levels()}); + FieldLayout layout_int ({ILEV},{grid->get_num_vertical_levels()+1}); const auto units = ekat::units::Units::nondimensional(); auto lat = grid->create_geometry_data("lat" , grid->get_2d_scalar_layout(), units); auto lon = grid->create_geometry_data("lon" , grid->get_2d_scalar_layout(), units); auto hyam = grid->create_geometry_data("hyam" , layout_mid, units); auto hybm = grid->create_geometry_data("hybm" , layout_mid, units); + auto hyai = grid->create_geometry_data("hyai" , layout_int, units); + auto hybi = grid->create_geometry_data("hybi" , layout_int, units); auto lev = grid->create_geometry_data("lev" , layout_mid, units); + auto ilev = grid->create_geometry_data("ilev" , layout_int, units); lat.deep_copy(ekat::ScalarTraits::invalid()); lon.deep_copy(ekat::ScalarTraits::invalid()); hyam.deep_copy(ekat::ScalarTraits::invalid()); hybm.deep_copy(ekat::ScalarTraits::invalid()); lev.deep_copy(ekat::ScalarTraits::invalid()); + ilev.deep_copy(ekat::ScalarTraits::invalid()); lat.sync_to_dev(); lon.sync_to_dev(); hyam.sync_to_dev(); hybm.sync_to_dev(); lev.sync_to_dev(); + ilev.sync_to_dev(); } else if (geo_data_source=="IC_FILE"){ const auto& filename = m_params.get("ic_filename"); if (scorpio::has_var(filename,"lat") && @@ -173,7 +179,9 @@ add_geo_data (const nonconstgrid_ptr_type& grid) const } if (scorpio::has_var(filename,"hyam") && - scorpio::has_var(filename,"hybm")) { + scorpio::has_var(filename,"hybm") && + scorpio::has_var(filename,"hyai") && + scorpio::has_var(filename,"hybi") ) { load_vertical_coordinates(grid,filename); } } @@ -234,53 +242,71 @@ load_vertical_coordinates (const nonconstgrid_ptr_type& grid, const std::string& using namespace ekat::units; FieldLayout layout_mid ({LEV},{grid->get_num_vertical_levels()}); + FieldLayout layout_int ({ILEV},{grid->get_num_vertical_levels()+1}); Units nondim = Units::nondimensional(); Units mbar (100*Pa,"mb"); auto hyam = grid->create_geometry_data("hyam", layout_mid, nondim); auto hybm = grid->create_geometry_data("hybm", layout_mid, nondim); + auto hyai = grid->create_geometry_data("hyai", layout_int, nondim); + auto hybi = grid->create_geometry_data("hybi", layout_int, nondim); auto lev = grid->create_geometry_data("lev", layout_mid, mbar); + auto ilev = grid->create_geometry_data("ilev", layout_int, mbar); // Create host mirrors for reading in data std::map host_views = { { "hyam", hyam.get_view() }, - { "hybm", hybm.get_view() } + { "hybm", hybm.get_view() }, + { "hyai", hyai.get_view() }, + { "hybi", hybi.get_view() } }; // Store view layouts using namespace ShortFieldTagsNames; std::map layouts = { { "hyam", hyam.get_header().get_identifier().get_layout() }, - { "hybm", hybm.get_header().get_identifier().get_layout() } + { "hybm", hybm.get_header().get_identifier().get_layout() }, + { "hyai", hyai.get_header().get_identifier().get_layout() }, + { "hybi", hybi.get_header().get_identifier().get_layout() } }; // Read hyam/hybm into host views ekat::ParameterList vcoord_reader_pl; vcoord_reader_pl.set("Filename",filename); - vcoord_reader_pl.set>("Field Names",{"hyam","hybm"}); + vcoord_reader_pl.set>("Field Names",{"hyam","hybm","hyai","hybi"}); AtmosphereInput vcoord_reader(vcoord_reader_pl,grid, host_views, layouts); vcoord_reader.read_variables(); vcoord_reader.finalize(); - // Build lev from hyam and hybm + // Build lev and ilev from hyam and hybm, and ilev from hyai and hybi using PC = scream::physics::Constants; const Real ps0 = PC::P0; - auto hya_v = hyam.get_view(); - auto hyb_v = hybm.get_view(); - auto lev_v = lev.get_view(); - for (int ii=0;iiget_num_vertical_levels();ii++) { - lev_v(ii) = 0.01*ps0*(hya_v(ii)+hyb_v(ii)); + auto hyam_v = hyam.get_view(); + auto hybm_v = hybm.get_view(); + auto lev_v = lev.get_view(); + auto hyai_v = hyai.get_view(); + auto hybi_v = hybi.get_view(); + auto ilev_v = ilev.get_view(); + auto num_lev = grid->get_num_vertical_levels(); + for (int ii=0;ii(f,grid)->check(); EKAT_REQUIRE_MSG (nan_check.result==CheckResult::Pass, "ERROR! NaN values detected in " + f.name() + " field.\n" + nan_check.msg); diff --git a/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp b/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp index 7d6590c327a6..bd775f68e079 100644 --- a/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp +++ b/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp @@ -37,7 +37,13 @@ CoarseningRemapper (const grid_ptr_type& src_grid, const auto& src_data = src_grid->get_geometry_data(name); const auto& src_data_fid = src_data.get_header().get_identifier(); const auto& layout = src_data_fid.get_layout(); - if (layout.tags()[0]!=COL) { + if (layout.tags().empty()) { + // This is a scalar field, so won't be coarsened. + // Simply copy it in the tgt grid, but we still need to assign the new grid name. + FieldIdentifier tgt_data_fid(src_data_fid.name(),src_data_fid.get_layout(),src_data_fid.get_units(),m_tgt_grid->name()); + auto tgt_data = m_coarse_grid->create_geometry_data(tgt_data_fid); + tgt_data.deep_copy(src_data); + } else if (layout.tags()[0]!=COL) { // Not a field to be coarsened (perhaps a vertical coordinate field). // Simply copy it in the tgt grid, but we still need to assign the new grid name. FieldIdentifier tgt_data_fid(src_data_fid.name(),src_data_fid.get_layout(),src_data_fid.get_units(),m_tgt_grid->name()); diff --git a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp index 65da6f2447ae..0dc73499ce6a 100644 --- a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp +++ b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp @@ -183,7 +183,7 @@ do_register_field (const identifier_type& src, const identifier_type& tgt) { constexpr auto COL = ShortFieldTagsNames::COL; EKAT_REQUIRE_MSG (src.get_layout().has_tag(COL), - "Error! Cannot register a field without COL tag in RefiningRemapperP2P.\n" + "Error! Cannot register a field without COL tag in HorizInterpRemapperBase.\n" " - field name: " + src.name() + "\n" " - field layout: " + src.get_layout().to_string() + "\n"); m_src_fields.push_back(field_type(src)); @@ -194,11 +194,11 @@ void HorizInterpRemapperBase:: do_bind_field (const int ifield, const field_type& src, const field_type& tgt) { EKAT_REQUIRE_MSG (src.data_type()==DataType::RealType, - "Error! RefiningRemapperRMA only allows fields with RealType data.\n" + "Error! HorizInterpRemapperBase only allows fields with RealType data.\n" " - src field name: " + src.name() + "\n" " - src field type: " + e2str(src.data_type()) + "\n"); EKAT_REQUIRE_MSG (tgt.data_type()==DataType::RealType, - "Error! RefiningRemapperRMA only allows fields with RealType data.\n" + "Error! HorizInterpRemapperBase only allows fields with RealType data.\n" " - tgt field name: " + tgt.name() + "\n" " - tgt field type: " + e2str(tgt.data_type()) + "\n"); @@ -352,7 +352,7 @@ local_mat_vec (const Field& x, const Field& y) const } default: { - EKAT_ERROR_MSG("Error::refining_remapper::local_mat_vec doesn't support fields of rank 4 or greater"); + EKAT_ERROR_MSG("[HorizInterpRemapperBase::local_mat_vec] Error! Fields of rank 4 or greater are not supported.\n"); } } } diff --git a/components/eamxx/src/share/grid/remap/refining_remapper_p2p.cpp b/components/eamxx/src/share/grid/remap/refining_remapper_p2p.cpp index f2b9871053ef..5f9d91595ece 100644 --- a/components/eamxx/src/share/grid/remap/refining_remapper_p2p.cpp +++ b/components/eamxx/src/share/grid/remap/refining_remapper_p2p.cpp @@ -52,7 +52,7 @@ void RefiningRemapperP2P::do_remap_fwd () for (int i=0; i; + constexpr auto COL = ShortFieldTagsNames::COL; + auto export_pids = m_imp_exp->export_pids(); auto export_lids = m_imp_exp->export_lids(); auto ncols_send = m_imp_exp->num_exports_per_pid(); @@ -174,6 +178,10 @@ void RefiningRemapperP2P::pack_and_send () for (int ifield=0; ifieldtgt later + continue; + } const auto f_col_sizes_scan_sum = m_fields_col_sizes_scan_sum[ifield]; switch (fl.rank()) { case 1: @@ -308,6 +316,8 @@ void RefiningRemapperP2P::recv_and_unpack () using TeamMember = typename KT::MemberType; using ESU = ekat::ExeSpaceUtils; + constexpr auto COL = ShortFieldTagsNames::COL; + auto import_pids = m_imp_exp->import_pids(); auto import_lids = m_imp_exp->import_lids(); auto ncols_recv = m_imp_exp->num_imports_per_pid(); @@ -318,6 +328,10 @@ void RefiningRemapperP2P::recv_and_unpack () for (int ifield=0; ifieldtgt later + continue; + } const auto f_col_sizes_scan_sum = m_fields_col_sizes_scan_sum[ifield]; switch (fl.rank()) { case 1: diff --git a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp index 9b524cec5e49..cb55080da384 100644 --- a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp +++ b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp @@ -17,175 +17,229 @@ namespace scream { +std::shared_ptr VerticalRemapper:: -VerticalRemapper (const grid_ptr_type& src_grid, - const std::string& map_file, - const Field& pmid_src, - const Field& pint_src) - : VerticalRemapper(src_grid,map_file,pmid_src,pint_src,constants::DefaultFillValue::value) -{ - // Nothing to do here -} - -VerticalRemapper:: -VerticalRemapper (const grid_ptr_type& src_grid, - const std::string& map_file, - const Field& pmid_src, - const Field& pint_src, - const Real mask_val) - : AbstractRemapper() - , m_comm (src_grid->get_comm()) - , m_mask_val(mask_val) +create_tgt_grid (const grid_ptr_type& src_grid, + const std::string& map_file) { - using namespace ShortFieldTagsNames; - - // Sanity checks - EKAT_REQUIRE_MSG (src_grid->type()==GridType::Point, - "Error! VerticalRemapper only works on PointGrid grids.\n" - " - src grid name: " + src_grid->name() + "\n" - " - src_grid_type: " + e2str(src_grid->type()) + "\n"); - EKAT_REQUIRE_MSG (src_grid->is_unique(), - "Error! VerticalRemapper requires a unique source grid.\n"); - - // This is a vertical remapper. We only go in one direction - m_bwd_allowed = false; - - // Create tgt_grid that is a clone of the src grid but with - // the correct number of levels. Note that when vertically - // remapping the target field will be defined on the same DOFs - // as the source field, but will have a different number of - // vertical levels. + // Create tgt_grid as a clone of src_grid with different nlevs scorpio::register_file(map_file,scorpio::FileMode::Read); auto nlevs_tgt = scorpio::get_dimlen(map_file,"lev"); auto tgt_grid = src_grid->clone("vertical_remap_tgt_grid",true); tgt_grid->reset_num_vertical_lev(nlevs_tgt); - this->set_grids(src_grid,tgt_grid); - - // Set the LEV and ILEV vertical profiles for interpolation from - set_source_pressure_fields(pmid_src,pint_src); // Gather the pressure level data for vertical remapping - set_pressure_levels(map_file); + auto layout = tgt_grid->get_vertical_layout(true); + Field p_tgt(FieldIdentifier("p_levs",layout,ekat::units::Pa,tgt_grid->name())); + p_tgt.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); + p_tgt.allocate_view(); + scorpio::read_var(map_file,"p_levs",p_tgt.get_view().data()); + p_tgt.sync_to_dev(); // Add tgt pressure levels to the tgt grid - tgt_grid->set_geometry_data(m_tgt_pressure); + tgt_grid->set_geometry_data(p_tgt); scorpio::release_file(map_file); + + return tgt_grid; +} + +VerticalRemapper:: +VerticalRemapper (const grid_ptr_type& src_grid, + const std::string& map_file) + : VerticalRemapper(src_grid,create_tgt_grid(src_grid,map_file)) +{ + set_target_pressure (m_tgt_grid->get_geometry_data("p_levs"),Both); +} + +VerticalRemapper:: +VerticalRemapper (const grid_ptr_type& src_grid, + const grid_ptr_type& tgt_grid) +{ + // We only go in one direction for simplicity, since we need to setup some + // infrsatructures, and we don't want to setup 2x as many "just in case". + // If you need to remap bwd, just create another remapper with src/tgt grids swapped. + m_bwd_allowed = false; + + EKAT_REQUIRE_MSG (src_grid->get_2d_scalar_layout().congruent(tgt_grid->get_2d_scalar_layout()), + "Error! Source and target grid can only differ for their number of level.\n"); + + this->set_grids (src_grid,tgt_grid); } FieldLayout VerticalRemapper:: create_src_layout (const FieldLayout& tgt_layout) const { - using namespace ShortFieldTagsNames; - EKAT_REQUIRE_MSG (is_valid_tgt_layout(tgt_layout), "[VerticalRemapper] Error! Input target layout is not valid for this remapper.\n" " - input layout: " + tgt_layout.to_string()); - return create_layout(tgt_layout,m_src_grid); + EKAT_REQUIRE_MSG (not m_tgt_mid_same_as_int, + "[VerticalRemapper::create_src_layout] Error! Cannot deduce source layout.\n" + " The target layout does not distinguish between LEV and ILEV.\n"); + + const auto& mid_layout = m_src_pmid.get_header().get_identifier().get_layout(); + const auto& int_layout = m_src_pint.get_header().get_identifier().get_layout(); + return create_layout(tgt_layout,m_src_grid,mid_layout.congruent(int_layout)); } FieldLayout VerticalRemapper:: create_tgt_layout (const FieldLayout& src_layout) const { - using namespace ShortFieldTagsNames; - EKAT_REQUIRE_MSG (is_valid_src_layout(src_layout), - "[VerticalRemapper] Error! Input source layout is not valid for this remapper.\n" + "[VerticalRemapper::create_tgt_layout] Error! Input source layout is not valid for this remapper.\n" " - input layout: " + src_layout.to_string()); - return create_layout(src_layout,m_tgt_grid); + EKAT_REQUIRE_MSG (not m_src_mid_same_as_int, + "[VerticalRemapper::create_tgt_layout] Error! Cannot deduce target layout.\n" + " The source layout does not distinguish between LEV and ILEV.\n"); + + const auto& mid_layout = m_tgt_pmid.get_header().get_identifier().get_layout(); + const auto& int_layout = m_tgt_pint.get_header().get_identifier().get_layout(); + return create_layout(src_layout,m_tgt_grid,mid_layout.congruent(int_layout)); } FieldLayout VerticalRemapper:: -create_layout (const FieldLayout& fl_in, - const grid_ptr_type& grid_out) const +create_layout (const FieldLayout& from_layout, + const std::shared_ptr& to_grid, + const bool int_same_as_mid) const { - // NOTE: for the vert remapper, it doesn't really make sense to distinguish - // between midpoints and interfaces: we're simply asking for a quantity - // at a given set of pressure levels. So we choose to have fl_out - // to *always* have LEV as vertical tag. - auto fl_out = FieldLayout::invalid(); - switch (fl_in.type()) { + using namespace ShortFieldTagsNames; + + auto to_layout = FieldLayout::invalid(); + bool midpoints; + std::string vdim_name; + switch (from_layout.type()) { case LayoutType::Scalar0D: [[ fallthrough ]]; case LayoutType::Vector0D: [[ fallthrough ]]; case LayoutType::Scalar2D: [[ fallthrough ]]; case LayoutType::Vector2D: [[ fallthrough ]]; case LayoutType::Tensor2D: // These layouts do not have vertical dim tags, so no change - fl_out = fl_in; + to_layout = from_layout; break; case LayoutType::Scalar1D: - fl_out = grid_out->get_vertical_layout(true); + midpoints = int_same_as_mid || from_layout.tags().back()==LEV; + to_layout = to_grid->get_vertical_layout(midpoints); break; case LayoutType::Scalar3D: - fl_out = grid_out->get_3d_scalar_layout(true); + midpoints = int_same_as_mid || from_layout.tags().back()==LEV; + to_layout = to_grid->get_3d_scalar_layout(midpoints); break; case LayoutType::Vector3D: - fl_out = grid_out->get_3d_vector_layout(true,fl_in.get_vector_dim()); + vdim_name = from_layout.name(from_layout.get_vector_component_idx()); + midpoints = int_same_as_mid || from_layout.tags().back()==LEV; + to_layout = to_grid->get_3d_vector_layout(midpoints,from_layout.get_vector_dim(),vdim_name); break; default: // NOTE: this also include Tensor3D. We don't really have any atm proc // that needs to handle a tensor3d quantity, so no need to add it EKAT_ERROR_MSG ( "[VerticalRemapper] Error! Layout not supported by VerticalRemapper.\n" - " - input layout: " + fl_in.to_string() + "\n"); + " - input layout: " + from_layout.to_string() + "\n"); } - return fl_out; + return to_layout; } void VerticalRemapper:: -set_pressure_levels(const std::string& map_file) +set_extrapolation_type (const ExtrapType etype, const TopBot where) { - // Ensure each map file gets a different decomp name - static std::map file2idx; - if (file2idx.find(map_file)==file2idx.end()) { - file2idx[map_file] = file2idx.size(); + if (where & Top) { + m_etype_top = etype; + } + if (where & Bot) { + m_etype_bot = etype; } +} - using namespace ShortFieldTagsNames; - auto layout = m_tgt_grid->get_vertical_layout(true); - FieldIdentifier fid("p_levs",layout,ekat::units::Pa,m_tgt_grid->name()); - m_tgt_pressure = Field(fid); - // Just in case input fields are packed - m_tgt_pressure.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); - m_tgt_pressure.allocate_view(); +void VerticalRemapper:: +set_mask_value (const Real mask_val) +{ + EKAT_REQUIRE_MSG (not ekat::is_invalid(mask_val), + "[VerticalRemapper::set_mask_value] Error! Input mask value must be a valid number.\n"); - auto remap_pres_data = m_tgt_pressure.get_view().data(); - scorpio::read_var(map_file,"p_levs",remap_pres_data); + m_mask_val = mask_val; +} + +void VerticalRemapper:: +set_source_pressure (const Field& p, const ProfileType ptype) +{ + set_pressure (p, "source", ptype); +} - m_tgt_pressure.sync_to_dev(); +void VerticalRemapper:: +set_target_pressure (const Field& p, const ProfileType ptype) +{ + set_pressure (p, "target", ptype); } void VerticalRemapper:: -set_source_pressure_fields(const Field& pmid, const Field& pint) +set_pressure (const Field& p, const std::string& src_or_tgt, const ProfileType ptype) { using namespace ShortFieldTagsNames; + using PackT = ekat::Pack; + + bool src = src_or_tgt=="source"; + + std::string msg_prefix = "[VerticalRemapper::set_" + src_or_tgt + "_pressure] "; - EKAT_REQUIRE_MSG(pmid.is_allocated(), - "Error! Source midpoint pressure field is not yet allocated.\n" - " - field name: " + pmid.name() + "\n"); - - EKAT_REQUIRE_MSG(pint.is_allocated(), - "Error! Source interface pressure field is not yet allocated.\n" - " - field name: " + pint.name() + "\n"); - - const auto& pmid_layout = pmid.get_header().get_identifier().get_layout(); - const auto& pint_layout = pint.get_header().get_identifier().get_layout(); - EKAT_REQUIRE_MSG(pmid_layout.congruent(m_src_grid->get_3d_scalar_layout(true)), - "Error! Source midpoint pressure field has the wrong layout.\n" - " - field name: " + pmid.name() + "\n" - " - field layout: " + pmid_layout.to_string() + "\n" - " - expected layout: " + m_src_grid->get_3d_scalar_layout(true).to_string() + "\n"); - EKAT_REQUIRE_MSG(pint_layout.congruent(m_src_grid->get_3d_scalar_layout(false)), - "Error! Source interface pressure field has the wrong layout.\n" - " - field name: " + pint.name() + "\n" - " - field layout: " + pint_layout.to_string() + "\n" - " - expected layout: " + m_src_grid->get_3d_scalar_layout(false).to_string() + "\n"); - - m_src_pmid = pmid; - m_src_pint = pint; + EKAT_REQUIRE_MSG(p.is_allocated(), + msg_prefix + "Field is not yet allocated.\n" + " - field name: " + p.name() + "\n"); + + EKAT_REQUIRE_MSG(p.get_header().get_alloc_properties().is_compatible(), + msg_prefix + "Field not compatible with default pack size.\n" + " - pack size: " + std::to_string(SCREAM_PACK_SIZE) + "\n"); + + const int nlevs = src ? m_src_grid->get_num_vertical_levels() + : m_tgt_grid->get_num_vertical_levels(); + const auto& p_layout = p.get_header().get_identifier().get_layout(); + const auto vtag = p_layout.tags().back(); + const auto vdim = p_layout.dims().back(); + + FieldTag expected_tag; + int expected_dim; + switch (ptype) { + case Midpoints: + expected_tag = LEV; + expected_dim = nlevs; + if (src) { + m_src_pmid = p; + } else { + m_tgt_pmid = p; + } + break; + case Interfaces: + expected_tag = ILEV; + expected_dim = nlevs+1; + if (src) { + m_src_pint = p; + } else { + m_tgt_pint = p; + } + break; + case Both: + expected_tag = LEV; + expected_dim = nlevs; + if (src) { + m_src_pint = p; + m_src_pmid = p; + m_src_mid_same_as_int = true; + } else { + m_tgt_pint = p; + m_tgt_pmid = p; + m_tgt_mid_same_as_int = true; + } + break; + default: + EKAT_ERROR_MSG ("[VerticalRemapper::set_source_pressure] Error! Unrecognized value for 'ptype'.\n"); + } + EKAT_REQUIRE_MSG (vtag==expected_tag and vdim==expected_dim, + msg_prefix + "Invalid pressure layout.\n" + " - layout: " + p_layout.to_string() + "\n" + " - expected last layout tag: " + e2str(expected_tag) + "\n" + " - expected last layout dim: " + std::to_string(expected_dim) + "\n"); } void VerticalRemapper:: @@ -198,7 +252,7 @@ do_register_field (const identifier_type& src, const identifier_type& tgt) // could have src with ILEV and tgt with LEV) auto src_layout = src.get_layout().clone(); auto tgt_layout = tgt.get_layout().clone(); - EKAT_REQUIRE_MSG(src_layout.strip_dims({ILEV,LEV}).congruent(tgt_layout.strip_dims({LEV})), + EKAT_REQUIRE_MSG(src_layout.strip_dims({ILEV,LEV}).congruent(tgt_layout.strip_dims({LEV,ILEV})), "[VerticalRemapper] Error! Once vertical level tag is stripped, src/tgt layouts are incompatible.\n" " - src field name: " + src.name() + "\n" " - tgt field name: " + tgt.name() + "\n" @@ -224,61 +278,66 @@ do_bind_field (const int ifield, const field_type& src, const field_type& tgt) auto& f_tgt = m_tgt_fields[ifield]; // Nonconst, since we need to set extra data in the header if (src_layout.has_tag(LEV) or src_layout.has_tag(ILEV)) { // Determine if this field can be handled with packs, and whether it's at midpoints + // NOTE: we don't know if mid==int on src or tgt. If it is, we use the other to determine mid-vs-int // Add mask tracking to the target field. The mask tracks location of tgt pressure levs that are outside the // bounds of the src pressure field, and hence cannot be recovered by interpolation auto& ft = m_field2type[src.name()]; - ft.midpoints = src.get_header().get_identifier().get_layout().has_tag(LEV); + ft.midpoints = m_src_mid_same_as_int + ? tgt.get_header().get_identifier().get_layout().has_tag(LEV) + : src.get_header().get_identifier().get_layout().has_tag(LEV); ft.packed = src.get_header().get_alloc_properties().is_compatible() and tgt.get_header().get_alloc_properties().is_compatible(); - // NOTE: for now we assume that masking is determined only by the COL,LEV location in space - // and that fields with multiple components will have the same masking for each component - // at a specific COL,LEV - src_layout.strip_dims({CMP}); + if (m_etype_top==Mask or m_etype_bot==Mask) { + // NOTE: for now we assume that masking is determined only by the COL,LEV location in space + // and that fields with multiple components will have the same masking for each component + // at a specific COL,LEV + src_layout.strip_dims({CMP}); - // I this mask has already been created, retrieve it, otherwise create it - const auto mask_name = m_tgt_grid->name() + "_" + ekat::join(src_layout.names(),"_") + "_mask"; - Field tgt_mask; - if (m_field2type.count(mask_name)==0) { - auto nondim = ekat::units::Units::nondimensional(); - // Create this src/tgt mask fields, and assign them to these src/tgt fields extra data + // I this mask has already been created, retrieve it, otherwise create it + const auto mask_name = m_tgt_grid->name() + "_" + ekat::join(src_layout.names(),"_") + "_mask"; + Field tgt_mask; + if (m_field2type.count(mask_name)==0) { + auto nondim = ekat::units::Units::nondimensional(); + // Create this src/tgt mask fields, and assign them to these src/tgt fields extra data - FieldIdentifier src_mask_fid (mask_name, src_layout, nondim, m_src_grid->name() ); - FieldIdentifier tgt_mask_fid = create_tgt_fid(src_mask_fid); + FieldIdentifier src_mask_fid (mask_name, src_layout, nondim, m_src_grid->name() ); + FieldIdentifier tgt_mask_fid = create_tgt_fid(src_mask_fid); - Field src_mask (src_mask_fid); - src_mask.allocate_view(); + Field src_mask (src_mask_fid); + src_mask.allocate_view(); - tgt_mask = Field (tgt_mask_fid); - tgt_mask.allocate_view(); + tgt_mask = Field (tgt_mask_fid); + tgt_mask.allocate_view(); - // Initialize the src mask values to 1.0 - src_mask.deep_copy(1.0); + // Initialize the src mask values to 1.0 + src_mask.deep_copy(1.0); - m_src_masks.push_back(src_mask); - m_tgt_masks.push_back(tgt_mask); + m_src_masks.push_back(src_mask); + m_tgt_masks.push_back(tgt_mask); - auto& mt = m_field2type[src_mask_fid.name()]; - mt.packed = false; - mt.midpoints = src_layout.has_tag(LEV); - } else { - for (size_t i=0; i void VerticalRemapper:: setup_lin_interp (const ekat::LinInterp& lin_interp, - const Field& p_src) const + const Field& p_src, const Field& p_tgt) const { using LI_t = ekat::LinInterp; using ESU = ekat::ExeSpaceUtils; using PackT = ekat::Pack; - auto p_src_v = p_src.get_view(); - auto p_tgt_v = m_tgt_pressure.get_view(); + using view2d = typename KokkosTypes::view; + using view1d = typename KokkosTypes::view; + + auto src1d = p_src.rank()==1; + auto tgt1d = p_tgt.rank()==1; + + view2d p_src2d_v, p_tgt2d_v; + view1d p_src1d_v, p_tgt1d_v; + if (src1d) { + p_src1d_v = p_src.get_view(); + } else { + p_src2d_v = p_src.get_view(); + } + if (tgt1d) { + p_tgt1d_v = p_tgt.get_view(); + } else { + p_tgt2d_v = p_tgt.get_view(); + } auto lambda = KOKKOS_LAMBDA(typename LI_t::MemberType const& team) { const int icol = team.league_rank(); - lin_interp.setup(team,ekat::subview(p_src_v,icol), - p_tgt_v); + // Extract subviews if src/tgt were not 1d to start with + auto x_src = p_src1d_v; + if (not src1d) + x_src = ekat::subview(p_src2d_v,icol); + auto x_tgt = p_tgt1d_v; + if (not tgt1d) + x_tgt = ekat::subview(p_tgt2d_v,icol); + + lin_interp.setup(team,x_src,x_tgt); }; - const int ncols = m_src_grid->get_num_local_dofs(); const int nlevs_tgt = m_tgt_grid->get_num_vertical_levels(); const int npacks_tgt = ekat::PackInfo::num_packs(nlevs_tgt); @@ -465,8 +550,7 @@ template void VerticalRemapper:: apply_vertical_interpolation(const ekat::LinInterp& lin_interp, const Field& f_src, const Field& f_tgt, - const Field& p_src, - const Real mask_val) const + const Field& p_src, const Field& p_tgt) const { // Note: if Packsize==1, we grab packs of size 1, which are for sure // compatible with the allocation @@ -474,44 +558,51 @@ apply_vertical_interpolation(const ekat::LinInterp& lin_interp, using PackT = ekat::Pack; using ESU = ekat::ExeSpaceUtils; - auto p_src_v = p_src.get_view(); - auto x_tgt = m_tgt_pressure.get_view(); - const auto& f_src_l = f_src.get_header().get_identifier().get_layout(); + using view2d = typename KokkosTypes::view; + using view1d = typename KokkosTypes::view; + + auto src1d = p_src.rank()==1; + auto tgt1d = p_tgt.rank()==1; + + view2d p_src2d_v, p_tgt2d_v; + view1d p_src1d_v, p_tgt1d_v; + if (src1d) { + p_src1d_v = p_src.get_view(); + } else { + p_src2d_v = p_src.get_view(); + } + if (tgt1d) { + p_tgt1d_v = p_tgt.get_view(); + } else { + p_tgt2d_v = p_tgt.get_view(); + } + + const auto& f_tgt_l = f_tgt.get_header().get_identifier().get_layout(); const int ncols = m_src_grid->get_num_local_dofs(); - const int nlevs_tgt = m_tgt_grid->get_num_vertical_levels(); - const int nlevs_src = f_src_l.dims().back(); + const int nlevs_tgt = f_tgt_l.dims().back(); const int npacks_tgt = ekat::PackInfo::num_packs(nlevs_tgt); - const int last_src_pack_idx = ekat::PackInfo::last_pack_idx(nlevs_src); - const int last_src_pack_end = ekat::PackInfo::last_vec_end(nlevs_src); - switch(f_src.rank()) { case 2: { auto f_src_v = f_src.get_view(); auto f_tgt_v = f_tgt.get_view< PackT**>(); auto policy = ESU::get_default_team_policy(ncols,npacks_tgt); - auto lambda = KOKKOS_LAMBDA(typename LI_t::MemberType const& team) { - - // Interpolate + auto lambda = KOKKOS_LAMBDA(typename LI_t::MemberType const& team) + { const int icol = team.league_rank(); - auto x_src = ekat::subview(p_src_v,icol); + + // Extract subviews if src/tgt pressures were not 1d to start with + auto x_src = p_src1d_v; + auto x_tgt = p_tgt1d_v; + if (not src1d) + x_src = ekat::subview(p_src2d_v,icol); + if (not tgt1d) + x_tgt = ekat::subview(p_tgt2d_v,icol); + auto y_src = ekat::subview(f_src_v,icol); auto y_tgt = ekat::subview(f_tgt_v,icol); lin_interp.lin_interp(team,x_src,x_tgt,y_src,y_tgt,icol); - team.team_barrier(); - - // If x_tgt is extrapolated, set to mask_val - auto x_min = x_src[0][0]; - auto x_max = x_src[last_src_pack_idx][last_src_pack_end-1]; - auto set_mask = [&](const int ipack) { - auto in_range = ekat::range(ipack*Packsize) < nlevs_tgt; - auto oob = (x_tgt[ipack]x_max) and in_range; - if (oob.any()) { - y_tgt[ipack].set(oob,mask_val); - } - }; - Kokkos::parallel_for (Kokkos::TeamThreadRange(team,npacks_tgt), set_mask); }; Kokkos::parallel_for("VerticalRemapper::apply_vertical_interpolation",policy,lambda); break; @@ -520,31 +611,25 @@ apply_vertical_interpolation(const ekat::LinInterp& lin_interp, { auto f_src_v = f_src.get_view(); auto f_tgt_v = f_tgt.get_view< PackT***>(); - const auto& layout = f_src.get_header().get_identifier().get_layout(); - const int ncomps = layout.get_vector_dim(); + const int ncomps = f_tgt_l.get_vector_dim(); auto policy = ESU::get_default_team_policy(ncols*ncomps,npacks_tgt); auto lambda = KOKKOS_LAMBDA(typename LI_t::MemberType const& team) { - // Interpolate const int icol = team.league_rank() / ncomps; const int icmp = team.league_rank() % ncomps; - auto x_src = ekat::subview(p_src_v,icol); + + // Extract subviews if src/tgt pressures were not 1d to start with + auto x_src = p_src1d_v; + auto x_tgt = p_tgt1d_v; + if (not src1d) + x_src = ekat::subview(p_src2d_v,icol); + if (not tgt1d) + x_tgt = ekat::subview(p_tgt2d_v,icol); + auto y_src = ekat::subview(f_src_v,icol,icmp); auto y_tgt = ekat::subview(f_tgt_v,icol,icmp); lin_interp.lin_interp(team,x_src,x_tgt,y_src,y_tgt,icol); - team.team_barrier(); - - // If x_tgt is extrapolated, set to mask_val - auto x_min = x_src[0][0]; - auto x_max = x_src[last_src_pack_idx][last_src_pack_end-1]; - auto set_mask = [&](const int ipack) { - auto oob = x_tgt[ipack]x_max; - if (oob.any()) { - y_tgt[ipack].set(oob,mask_val); - } - }; - Kokkos::parallel_for (Kokkos::TeamThreadRange(team,npacks_tgt), set_mask); }; Kokkos::parallel_for("VerticalRemapper::apply_vertical_interpolation",policy,lambda); break; @@ -557,4 +642,151 @@ apply_vertical_interpolation(const ekat::LinInterp& lin_interp, } } +void VerticalRemapper:: +extrapolate (const Field& f_src, + const Field& f_tgt, + const Field& p_src, + const Field& p_tgt, + const Real mask_val) const +{ + using ESU = ekat::ExeSpaceUtils; + + using view2d = typename KokkosTypes::view; + using view1d = typename KokkosTypes::view; + + auto src1d = p_src.rank()==1; + auto tgt1d = p_tgt.rank()==1; + + view2d p_src2d_v, p_tgt2d_v; + view1d p_src1d_v, p_tgt1d_v; + if (src1d) { + p_src1d_v = p_src.get_view(); + } else { + p_src2d_v = p_src.get_view(); + } + if (tgt1d) { + p_tgt1d_v = p_tgt.get_view(); + } else { + p_tgt2d_v = p_tgt.get_view(); + } + + const auto& f_tgt_l = f_tgt.get_header().get_identifier().get_layout(); + const auto& f_src_l = f_src.get_header().get_identifier().get_layout(); + const int ncols = m_src_grid->get_num_local_dofs(); + const int nlevs_tgt = f_tgt_l.dims().back(); + const int nlevs_src = f_src_l.dims().back(); + + auto etop = m_etype_top; + auto ebot = m_etype_bot; + auto mid = nlevs_tgt / 2; + switch(f_src.rank()) { + case 2: + { + auto f_src_v = f_src.get_view(); + auto f_tgt_v = f_tgt.get_view< Real**>(); + auto policy = ESU::get_default_team_policy(ncols,nlevs_tgt); + + using MemberType = typename decltype(policy)::member_type; + auto lambda = KOKKOS_LAMBDA(const MemberType& team) + { + const int icol = team.league_rank(); + + // Extract subviews if src/tgt pressures were not 1d to start with + auto x_src = p_src1d_v; + auto x_tgt = p_tgt1d_v; + if (not src1d) + x_src = ekat::subview(p_src2d_v,icol); + if (not tgt1d) + x_tgt = ekat::subview(p_tgt2d_v,icol); + + auto y_src = ekat::subview(f_src_v,icol); + auto y_tgt = ekat::subview(f_tgt_v,icol); + + auto x_min = x_src[0]; + auto x_max = x_src[nlevs_src-1]; + auto extrapolate = [&](const int ilev) { + if (ilev>=mid) { + // Near surface + if (x_tgt[ilev]>x_max) { + if (ebot==P0) { + y_tgt[ilev] = y_src[nlevs_src-1]; + } else { + y_tgt[ilev] = mask_val; + } + } + } else { + // Near top + if (x_tgt[ilev](); + auto f_tgt_v = f_tgt.get_view< Real***>(); + const int ncomps = f_tgt_l.get_vector_dim(); + auto policy = ESU::get_default_team_policy(ncols*ncomps,nlevs_tgt); + + using MemberType = typename decltype(policy)::member_type; + auto lambda = KOKKOS_LAMBDA(const MemberType& team) + { + const int icol = team.league_rank() / ncomps; + const int icmp = team.league_rank() % ncomps; + + // Extract subviews if src/tgt pressures were not 1d to start with + auto x_src = p_src1d_v; + auto x_tgt = p_tgt1d_v; + if (not src1d) + x_src = ekat::subview(p_src2d_v,icol); + if (not tgt1d) + x_tgt = ekat::subview(p_tgt2d_v,icol); + + auto y_src = ekat::subview(f_src_v,icol,icmp); + auto y_tgt = ekat::subview(f_tgt_v,icol,icmp); + auto x_min = x_src[0]; + auto x_max = x_src[nlevs_src-1]; + auto extrapolate = [&](const int ilev) { + if (ilev>=mid) { + // Near surface + if (x_tgt[ilev]>x_max) { + if (ebot==P0) { + y_tgt[ilev] = y_src[nlevs_src-1]; + } else { + y_tgt[ilev] = mask_val; + } + } + } else { + // Near top + if (x_tgt[ilev] + create_tgt_grid (const grid_ptr_type& src_grid, + const std::string& map_file); + protected: - FieldLayout create_layout (const FieldLayout& fl_in, - const grid_ptr_type& grid_out) const; + void set_pressure (const Field& p, const std::string& src_or_tgt, const ProfileType ptype); + FieldLayout create_layout (const FieldLayout& from_layout, + const std::shared_ptr& to_grid, + const bool int_same_as_mid) const; const identifier_type& do_get_src_field_id (const int ifield) const override { return m_src_fields[ifield].get_header().get_identifier(); @@ -89,25 +134,23 @@ class VerticalRemapper : public AbstractRemapper EKAT_ERROR_MSG ("VerticalRemapper only supports fwd remapping.\n"); } - void set_pressure_levels (const std::string& map_file); - void do_print(); - #ifdef KOKKOS_ENABLE_CUDA public: #endif template void apply_vertical_interpolation (const ekat::LinInterp& lin_interp, const Field& f_src, const Field& f_tgt, - const Field& p_src, - const Real mask_value) const; + const Field& p_src, const Field& p_tgt) const; + void extrapolate (const Field& f_src, const Field& f_tgt, + const Field& p_src, const Field& p_tgt, + const Real mask_val) const; template void setup_lin_interp (const ekat::LinInterp& lin_interp, - const Field& p_src) const; + const Field& p_src, const Field& p_tgt) const; protected: - void set_source_pressure_fields(const Field& pmid, const Field& pint); void create_lin_interp (); using KT = KokkosTypes; @@ -122,14 +165,22 @@ class VerticalRemapper : public AbstractRemapper // Source and target fields std::vector m_src_fields; std::vector m_tgt_fields; - std::vector m_tgt_masks; std::vector m_src_masks; + std::vector m_tgt_masks; // Vertical profile fields, both for source and target - Real m_mask_val; - Field m_tgt_pressure; - Field m_src_pmid; // Src vertical profile for LEV layouts - Field m_src_pint; // Src vertical profile for ILEV layouts + Field m_src_pmid; + Field m_src_pint; + Field m_tgt_pmid; + Field m_tgt_pint; + + bool m_src_mid_same_as_int = false; + bool m_tgt_mid_same_as_int = false; + + // Extrapolation settings at top/bottom. Default to P0 extrapolation + ExtrapType m_etype_top = P0; + ExtrapType m_etype_bot = P0; + Real m_mask_val = std::numeric_limits::quiet_NaN(); // We need to remap mid/int fields separately, and we want to use packs if possible, // so we need to divide input fields into 4 separate categories diff --git a/components/eamxx/src/share/io/CMakeLists.txt b/components/eamxx/src/share/io/CMakeLists.txt index 6c24d3bc51da..4908e4ae0c95 100644 --- a/components/eamxx/src/share/io/CMakeLists.txt +++ b/components/eamxx/src/share/io/CMakeLists.txt @@ -59,7 +59,7 @@ add_library(scream_io scream_io_utils.cpp ) -target_link_libraries(scream_io PUBLIC scream_share scream_scorpio_interface) +target_link_libraries(scream_io PUBLIC scream_share scream_scorpio_interface diagnostics) if (NOT SCREAM_LIB_ONLY) add_subdirectory(tests) diff --git a/components/eamxx/src/share/io/scorpio_input.cpp b/components/eamxx/src/share/io/scorpio_input.cpp index e8a510130e03..f81bb09f8a37 100644 --- a/components/eamxx/src/share/io/scorpio_input.cpp +++ b/components/eamxx/src/share/io/scorpio_input.cpp @@ -46,6 +46,14 @@ AtmosphereInput (const std::string& filename, init(params,fm); } +AtmosphereInput:: +AtmosphereInput (const std::vector& fields_names, + const std::shared_ptr& grid) +{ + set_grid(grid); + m_fields_names = fields_names; +} + AtmosphereInput:: ~AtmosphereInput () { @@ -73,10 +81,10 @@ init (const ekat::ParameterList& params, // Sets the internal field mgr, and possibly sets up the remapper set_field_manager(field_mgr); + m_inited_with_fields = true; + // Init scorpio internal structures init_scorpio_structures (); - - m_inited_with_fields = true; } void AtmosphereInput:: @@ -113,10 +121,10 @@ init (const ekat::ParameterList& params, " layout = " + it.first); } + m_inited_with_views = true; + // Init scorpio internal structures init_scorpio_structures (); - - m_inited_with_views = true; } /* ---------------------------------------------------------- */ @@ -173,6 +181,28 @@ set_field_manager (const std::shared_ptr& field_mgr) } } +void AtmosphereInput:: +set_fields (const std::vector& fields) { + auto fm = std::make_shared(m_io_grid); + m_fields_names.clear(); + for (const auto& f : fields) { + fm->add_field(f); + m_fields_names.push_back(f.name()); + } + set_field_manager(fm); + m_inited_with_fields = true; +} + +void AtmosphereInput:: +reset_filename (const std::string& filename) +{ + if (m_filename!="") { + scorpio::release_file(m_filename); + } + m_params.set("Filename",filename); + m_filename = filename; + init_scorpio_structures(); +} /* ---------------------------------------------------------- */ void AtmosphereInput:: @@ -220,6 +250,7 @@ void AtmosphereInput::read_variables (const int time_index) // Read the data auto v1d = m_host_views_1d.at(name); + scorpio::read_var(m_filename,name,v1d.data(),time_index); // If we have a field manager, make sure the data is correctly @@ -350,6 +381,9 @@ void AtmosphereInput::finalize() /* ---------------------------------------------------------- */ void AtmosphereInput::init_scorpio_structures() { + EKAT_REQUIRE_MSG (m_inited_with_views or m_inited_with_fields, + "Error! Cannot init scorpio structures until fields/views have been set.\n"); + std::string iotype_str = m_params.get("iotype", "default"); auto iotype = scorpio::str2iotype(iotype_str); diff --git a/components/eamxx/src/share/io/scorpio_input.hpp b/components/eamxx/src/share/io/scorpio_input.hpp index 2c0a7e76c3eb..7a53f1e8063c 100644 --- a/components/eamxx/src/share/io/scorpio_input.hpp +++ b/components/eamxx/src/share/io/scorpio_input.hpp @@ -61,6 +61,10 @@ class AtmosphereInput const std::shared_ptr& grid, const std::vector& fields, const bool skip_grid_checks = false); + // This constructor only sets the minimal info, deferring initialization + // to when set_field_manager/reset_fields and reset_filename are called + AtmosphereInput (const std::vector& fields_names, + const std::shared_ptr& grid); // Due to resource acquisition (in scorpio), avoid copies AtmosphereInput (const AtmosphereInput&) = delete; @@ -98,9 +102,11 @@ class AtmosphereInput // Getters std::string get_filename() { return m_filename; } // Simple getter to query the filename for this stream. - // Expose the ability to set field manager for cases like time_interpolation where we swap fields - // between field managers to avoid deep_copy. + // Expose the ability to set/reset fields/field_manager for cases like data interpolation, + // where we swap pointers but all the scorpio data structures are unchanged. void set_field_manager (const std::shared_ptr& field_mgr); + void set_fields (const std::vector& fields); + void reset_filename (const std::string& filename); // Option to add a logger void set_logger(const std::shared_ptr& atm_logger) { diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index a105e244d76f..10b2ee23d866 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -6,6 +6,8 @@ #include "share/util/scream_timing.hpp" #include "share/field/field_utils.hpp" +#include "diagnostics/register_diagnostics.hpp" + #include "ekat/util/ekat_units.hpp" #include "ekat/util/ekat_string_utils.hpp" #include "ekat/std_meta/ekat_std_utils.hpp" @@ -227,9 +229,13 @@ AtmosphereOutput (const ekat::Comm& comm, const ekat::ParameterList& params, if (use_vertical_remap_from_file) { // We build a remapper, to remap fields from the fm grid to the io grid auto vert_remap_file = params.get("vertical_remap_file"); - auto f_lev = get_field("p_mid","sim"); - auto f_ilev = get_field("p_int","sim"); - m_vert_remapper = std::make_shared(io_grid,vert_remap_file,f_lev,f_ilev,m_fill_value); + auto p_mid = get_field("p_mid","sim"); + auto p_int = get_field("p_int","sim"); + auto vert_remapper = std::make_shared(io_grid,vert_remap_file); + vert_remapper->set_source_pressure (p_mid,p_int); + vert_remapper->set_mask_value(m_fill_value); + vert_remapper->set_extrapolation_type(VerticalRemapper::Mask); // both Top AND Bot + m_vert_remapper = vert_remapper; io_grid = m_vert_remapper->get_tgt_grid(); set_grid(io_grid); @@ -500,6 +506,19 @@ run (const std::string& filename, // then there's no point in copying from the field's view to dev_view if (not is_aliasing_field_view) { switch (rank) { + case 0: + { + auto new_view_0d = field.get_view(); + auto avg_view_0d = view_Nd_dev<0>(data); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int) { + if (do_avg_cnt) { + combine_and_fill(new_view_0d(),avg_view_0d(),avg_type,fill_value); + } else { + combine(new_view_0d(),avg_view_0d(),avg_type); + } + }); + break; + } case 1: { // For rank-1 views, we use strided layout, since it helps us @@ -1031,9 +1050,15 @@ register_variables(const std::string& filename, // Gather longname (if not already in the io: string attributes) if (str_atts.count("long_name")==0) { - auto longname = m_longnames.get_longname(name); + auto longname = m_default_metadata.get_longname(name); scorpio::set_attribute(filename, name, "long_name", longname); } + + // Gather standard name, CF-Compliant (if not already in the io: string attributes) + if (str_atts.count("standard_name")==0) { + auto standardname = m_default_metadata.get_standardname(name); + scorpio::set_attribute(filename, name, "standard_name", standardname); + } } } // Now register the average count variables @@ -1085,7 +1110,7 @@ AtmosphereOutput::get_var_dof_offsets(const FieldLayout& layout) // Precompute this *before* the early return, since it involves collectives. // If one rank owns zero cols, and returns prematurely, the others will be left waiting. - AbstractGrid::gid_type min_gid; + AbstractGrid::gid_type min_gid = -1; if (layout.has_tag(COL) or layout.has_tag(EL)) { min_gid = m_io_grid->get_global_min_dof_gid(); } @@ -1289,6 +1314,8 @@ get_field(const std::string& name, const std::string& mode) const } else { EKAT_ERROR_MSG ("ERROR::AtmosphereOutput::get_field Field " + name + " not found in " + mode + " field manager or diagnostics list."); } + static Field f; + return f; } /* ---------------------------------------------------------- */ void AtmosphereOutput::set_diagnostics() @@ -1311,137 +1338,31 @@ void AtmosphereOutput::set_diagnostics() /* ---------------------------------------------------------- */ std::shared_ptr -AtmosphereOutput::create_diagnostic (const std::string& diag_field_name) { - auto& diag_factory = AtmosphereDiagnosticFactory::instance(); +AtmosphereOutput::create_diagnostic (const std::string& diag_field_name) +{ + // We need scream scope resolution, since this->create_diagnostic is hiding it + auto diag = scream::create_diagnostic(diag_field_name,get_field_manager("sim")->get_grid()); - // Construct a diagnostic by this name - ekat::ParameterList params; - std::string diag_name; + // Some diags need some extra setup or trigger extra behaviors std::string diag_avg_cnt_name = ""; - - if (diag_field_name.find("_at_")!=std::string::npos) { - // The diagnostic must be one of - // - ${field_name}_at_lev_${N} <- interface fields still use "_lev_" - // - ${field_name}_at_model_bot - // - ${field_name}_at_model_top - // - ${field_name}_at_${M}X - // where M/N are numbers (N integer), X=Pa, hPa, mb, or m - auto tokens = ekat::split(diag_field_name,"_at_"); - EKAT_REQUIRE_MSG (tokens.size()==2, - "Error! Unexpected diagnostic name: " + diag_field_name + "\n"); - - const auto& fname = tokens.front(); - params.set("field_name",fname); - params.set("grid_name",get_field_manager("sim")->get_grid()->name()); - - params.set("vertical_location", tokens[1]); + auto& params = diag->get_params(); + if (diag->name()=="FieldAtPressureLevel") { params.set("mask_value",m_fill_value); - - // Conventions on notation (N=any integer): - // FieldAtLevel : var_at_lev_N, var_at_model_top, var_at_model_bot - // FieldAtPressureLevel: var_at_Nx, with x=mb,Pa,hPa - // FieldAtHeight : var_at_Nm_above_Y (Y=sealevel or surface) - if (tokens[1].find_first_of("0123456789.")==0) { - auto units_start = tokens[1].find_first_not_of("0123456789."); - auto units = tokens[1].substr(units_start); - if (units.find("_above_") != std::string::npos) { - // The field is at a height above a specific reference. - // Currently we only support FieldAtHeight above "sealevel" or "surface" - auto subtokens = ekat::split(units,"_above_"); - params.set("surface_reference",subtokens[1]); - units = subtokens[0]; - // Need to reset the vertical location to strip the "_above_" part of the string. - params.set("vertical_location", tokens[1].substr(0,units_start)+subtokens[0]); - // If the slice is "above_sealevel" then we need to track the avg cnt uniquely. - // Note, "above_surface" is expected to never have masking and can thus use - // the typical 2d layout avg cnt. - if (subtokens[1]=="sealevel") { - diag_avg_cnt_name = "_" + tokens[1]; // Set avg_cnt tracking for this specific slice - // If we have 2D slices we need to be tracking the average count, - // if m_avg_type is not Instant - m_track_avg_cnt = m_track_avg_cnt || m_avg_type!=OutputAvgType::Instant; - } - } - if (units=="m") { - diag_name = "FieldAtHeight"; - EKAT_REQUIRE_MSG(params.isParameter("surface_reference"),"Error! Output field request for " + diag_field_name + " is missing a surface reference." - " Please add either '_above_sealevel' or '_above_surface' to the field name"); - } else if (units=="mb" or units=="Pa" or units=="hPa") { - diag_name = "FieldAtPressureLevel"; - diag_avg_cnt_name = "_" + tokens[1]; // Set avg_cnt tracking for this specific slice - // If we have 2D slices we need to be tracking the average count, - // if m_avg_type is not Instant - m_track_avg_cnt = m_track_avg_cnt || m_avg_type!=OutputAvgType::Instant; - } else { - EKAT_ERROR_MSG ("Error! Invalid units x for 'field_at_Nx' diagnostic.\n"); - } - } else { - diag_name = "FieldAtLevel"; - } - } else if (diag_field_name=="precip_liq_surf_mass_flux" or - diag_field_name=="precip_ice_surf_mass_flux" or - diag_field_name=="precip_total_surf_mass_flux") { - diag_name = "precip_surf_mass_flux"; - // split will return [X, ''], with X being whatever is before '_surf_mass_flux' - auto type = ekat::split(diag_field_name.substr(7),"_surf_mass_flux").front(); - params.set("precip_type",type); - } else if (diag_field_name=="IceWaterPath" or - diag_field_name=="LiqWaterPath" or - diag_field_name=="RainWaterPath" or - diag_field_name=="RimeWaterPath" or - diag_field_name=="VapWaterPath") { - diag_name = "WaterPath"; - // split will return the list [X, ''], with X being whatever is before 'WaterPath' - params.set("Water Kind",ekat::split(diag_field_name,"WaterPath").front()); - } else if (diag_field_name=="IceNumberPath" or - diag_field_name=="LiqNumberPath" or - diag_field_name=="RainNumberPath") { - diag_name = "NumberPath"; - // split will return the list [X, ''], with X being whatever is before 'NumberPath' - params.set("Number Kind",ekat::split(diag_field_name,"NumberPath").front()); - } else if (diag_field_name=="AeroComCldTop" or - diag_field_name=="AeroComCldBot") { - diag_name = "AeroComCld"; - // split will return the list ['', X], with X being whatever is after 'AeroComCld' - params.set("AeroComCld Kind",ekat::split(diag_field_name,"AeroComCld").back()); - } else if (diag_field_name=="MeridionalVapFlux" or - diag_field_name=="ZonalVapFlux") { - diag_name = "VaporFlux"; - // split will return the list [X, ''], with X being whatever is before 'VapFlux' - params.set("Wind Component",ekat::split(diag_field_name,"VapFlux").front()); - } else if (diag_field_name.find("_atm_backtend")!=std::string::npos) { - diag_name = "AtmBackTendDiag"; - // Set the grid_name - params.set("grid_name",get_field_manager("sim")->get_grid()->name()); - // split will return [X, ''], with X being whatever is before '_atm_tend' - params.set("Tendency Name",ekat::split(diag_field_name,"_atm_backtend").front()); - } else if (diag_field_name=="PotentialTemperature" or - diag_field_name=="LiqPotentialTemperature") { - diag_name = "PotentialTemperature"; - if (diag_field_name == "LiqPotentialTemperature") { - params.set("Temperature Kind", "Liq"); - } else { - params.set("Temperature Kind", "Tot"); + diag_avg_cnt_name = "_" + + params.get("pressure_value") + + params.get("pressure_units"); + m_track_avg_cnt = m_track_avg_cnt || m_avg_type!=OutputAvgType::Instant; + } else if (diag->name()=="FieldAtHeight") { + if (params.get("surface_reference")=="sealevel") { + diag_avg_cnt_name = "_" + + params.get("height_value") + + params.get("height_units") + "_above_sealevel"; + m_track_avg_cnt = m_track_avg_cnt || m_avg_type!=OutputAvgType::Instant; } - } else { - diag_name = diag_field_name; - } - - // These fields are special case of VerticalLayer diagnostic. - // The diagnostics requires the name to be given as param value. - if (diag_name == "z_int" or diag_name == "z_mid" or - diag_name == "geopotential_int" or diag_name == "geopotential_mid" or - diag_name == "height_int" or diag_name == "height_mid" or - diag_name == "dz") { - params.set("diag_name", diag_name); } - // Create the diagnostic - auto diag = diag_factory.create(diag_name,m_comm,params); - diag->set_grids(m_grids_manager); - // Ensure there's an entry in the map for this diag, so .at(diag_name) always works - auto& deps = m_diag_depends_on_diags[diag->name()]; + auto& deps = m_diag_depends_on_diags[diag_field_name]; // Initialize the diagnostic const auto sim_field_mgr = get_field_manager("sim"); @@ -1452,12 +1373,13 @@ AtmosphereOutput::create_diagnostic (const std::string& diag_field_name) { if (m_diagnostics.count(fname)==0) { m_diagnostics[fname] = create_diagnostic(fname); } - auto dep = m_diagnostics.at(fname); deps.push_back(fname); } diag->set_required_field (get_field(fname,"sim")); } + diag->initialize(util::TimeStamp(),RunType::Initial); + // If specified, set avg_cnt tracking for this diagnostic. if (m_track_avg_cnt) { const auto diag_field = diag->get_diagnostic(); @@ -1480,6 +1402,17 @@ update_avg_cnt_view(const Field& field, view_1d_dev& dev_view) { KT::RangePolicy policy(0,layout.size()); const auto extents = layout.extents(); switch (layout.rank()) { + case 0: + { + auto src_view_0d = field.get_view(); + auto tgt_view_0d = view_Nd_dev<0>(data); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int) { + if (src_view_0d()!=fill_value) { + tgt_view_0d() += 1; + } + }); + break; + } case 1: { // For rank-1 views, we use strided layout, since it helps us diff --git a/components/eamxx/src/share/io/scorpio_output.hpp b/components/eamxx/src/share/io/scorpio_output.hpp index 2cb45f28a322..b1dd6b36cf3e 100644 --- a/components/eamxx/src/share/io/scorpio_output.hpp +++ b/components/eamxx/src/share/io/scorpio_output.hpp @@ -6,7 +6,8 @@ #include "share/field/field_manager.hpp" #include "share/grid/abstract_grid.hpp" #include "share/grid/grids_manager.hpp" -#include "share/util//scream_time_stamp.hpp" +#include "share/util/scream_time_stamp.hpp" +#include "share/util/scream_utils.hpp" #include "share/atm_process/atmosphere_diagnostic.hpp" #include "ekat/ekat_parameter_list.hpp" @@ -206,7 +207,7 @@ class AtmosphereOutput std::map> m_diagnostics; std::map> m_diag_depends_on_diags; std::map m_diag_computed; - LongNames m_longnames; + DefaultMetadata m_default_metadata; // Use float, so that if output fp_precision=float, this is a representable value. // Otherwise, you would get an error from Netcdf, like diff --git a/components/eamxx/src/share/io/scream_io_control.hpp b/components/eamxx/src/share/io/scream_io_control.hpp index 63cef4111991..07e60b4f24df 100644 --- a/components/eamxx/src/share/io/scream_io_control.hpp +++ b/components/eamxx/src/share/io/scream_io_control.hpp @@ -22,6 +22,12 @@ struct IOControl { util::TimeStamp next_write_ts; util::TimeStamp last_write_ts; + // At run time, set dt in the struct, so we can compute next_write_ts correctly, + // even if freq_units is "nsteps" + // NOTE: this ASSUMES dt is constant throughout the run (i.e., no time adaptivity). + // An error will be thrown if dt changes, so developers can fix this if we ever support variable dt + double dt = 0; + bool output_enabled () const { return frequency_units!="none" && frequency_units!="never"; } @@ -29,44 +35,78 @@ struct IOControl { bool is_write_step (const util::TimeStamp& ts) const { if (not output_enabled()) return false; return frequency_units=="nsteps" ? ts.get_num_steps()==next_write_ts.get_num_steps() - : ts==next_write_ts; + : (ts.get_date()==next_write_ts.get_date() and + ts.get_time()==next_write_ts.get_time()); + } + + void set_frequency_units (const std::string& freq_unit) { + if (freq_unit=="none" or freq_unit=="never") { + frequency_units = freq_unit; + } else if (freq_unit=="nstep" or freq_unit=="nsteps") { + frequency_units = "nsteps"; + } else if (freq_unit=="nsecond" or freq_unit=="nseconds" or freq_unit=="nsecs") { + frequency_units = "nsecs"; + } else if (freq_unit=="nminute" or freq_unit=="nminutes" or freq_unit=="nmins") { + frequency_units = "nmins"; + } else if (freq_unit=="nhour" or freq_unit=="nhours") { + frequency_units = "nhours"; + } else if (freq_unit=="nday" or freq_unit=="ndays") { + frequency_units = "ndays"; + } else if (freq_unit=="nmonth" or freq_unit=="nmonths") { + frequency_units = "nmonths"; + } else if (freq_unit=="nyear" or freq_unit=="nyears") { + frequency_units = "nyears"; + } else { + // TODO - add support for "end" as an option + EKAT_ERROR_MSG("Error! Unsupported frequency units of " + freq_unit + " provided."); + } + + } + + void set_dt (const double dt_in) { + EKAT_REQUIRE_MSG (dt==0 or dt==dt_in, + "[IOControl::set_dt] Error! Cannot reset dt once it is set.\n"); + + dt = dt_in; } // Computes next_write_ts from frequency and last_write_ts void compute_next_write_ts () { EKAT_REQUIRE_MSG (last_write_ts.is_valid(), "Error! Cannot compute next_write_ts, since last_write_ts was never set.\n"); + next_write_ts = last_write_ts; if (frequency_units=="nsteps") { - // This avoids having an invalid date/time in the above check next time this fcn runs - next_write_ts = last_write_ts; + // This avoids having an invalid/wrong date/time in StorageSpecs::snapshot_fits + // if storage type is NumSnaps + next_write_ts += dt*frequency; next_write_ts.set_num_steps(last_write_ts.get_num_steps()+frequency); } else if (frequency_units=="nsecs") { - next_write_ts = last_write_ts; next_write_ts += frequency; } else if (frequency_units=="nmins") { - next_write_ts = last_write_ts; next_write_ts += frequency*60; } else if (frequency_units=="nhours") { - next_write_ts = last_write_ts; next_write_ts += frequency*3600; } else if (frequency_units=="ndays") { - next_write_ts = last_write_ts; next_write_ts += frequency*86400; - } else if (frequency_units=="nmonths" or frequency_units=="nyears") { + } else if (frequency_units=="nmonths") { + auto date = last_write_ts.get_date(); + int temp = date[1] + frequency - 1; + date[1] = temp % 12 + 1; + date[0] += temp / 12; + + // NOTE: we MAY have moved to an invalid date. E.g., if last_write + // was on Mar 31st, and units='nmonths', date now points to Apr 31st. + // We fix this by adjusting the date to the last day of the month. + // HOWEVER, this means we will *always* write on the 30th of each month after then, + // since we have no memory of the fact that we were writing on the 31st before. + auto month_beg = util::TimeStamp({date[0],date[1],1},{0,0,0}); + auto last_day = month_beg.days_in_curr_month(); + date[2] = std::min(date[2],last_day); + + next_write_ts = util::TimeStamp(date,last_write_ts.get_time()); + } else if (frequency_units=="nyears") { auto date = last_write_ts.get_date(); - if (frequency_units=="nmonths") { - int temp = date[1] + frequency - 1; - date[1] = temp % 12 + 1; - date[0] += temp / 12; - } else { - date[0] += frequency; - } - - // Fix day, in case we moved to a month/year where current days. E.g., if last_write - // was on Mar 31st, and units='nmonths', next write is on Apr 30th. HOWEVER, this - // means we will *always* write on the 30th of each month after then, since we have - // no memory of the fact that we were writing on the 31st before. - date[2] = std::min(date[2],util::days_in_month(date[0],date[1])); + date[0] += frequency; next_write_ts = util::TimeStamp(date,last_write_ts.get_time()); } else { EKAT_ERROR_MSG ("Error! Unrecognized/unsupported frequency unit '" + frequency_units + "'\n"); diff --git a/components/eamxx/src/share/io/scream_io_file_specs.hpp b/components/eamxx/src/share/io/scream_io_file_specs.hpp index ec0e2f50d796..ae5a00ff55b0 100644 --- a/components/eamxx/src/share/io/scream_io_file_specs.hpp +++ b/components/eamxx/src/share/io/scream_io_file_specs.hpp @@ -14,10 +14,13 @@ namespace scream { // How the file capacity is specified +// NOTE: Keep Yearly=0, Monthly=1, Daily=2, so you can use them +// to access a TimeStamp date at the correct index in StorageSpecs methods enum StorageType { - NumSnaps, // Fixed number of snaps per file - Monthly, // Each file contains output for one month - Yearly // Each file contains output for one year + Yearly = 0, // Each file contains output for one year + Monthly = 1, // Each file contains output for one month + Daily = 2, // Each file contains output for one day + NumSnaps // Fixed number of snaps per file }; inline std::string e2str (const StorageType st) { @@ -25,6 +28,7 @@ inline std::string e2str (const StorageType st) { case NumSnaps: return "num_snapshots"; case Yearly: return "one_year"; case Monthly: return "one_month"; + case Daily: return "one_day"; default: return "unknown"; } } @@ -33,33 +37,38 @@ struct StorageSpecs { StorageType type = NumSnaps; - // Current index ***in terms of this->type*** - // If type==NumSnaps, curr_idx=num_snapshots_in_file, - // otherwise it is the month/year index stored in this file + // Current index for type!=NumSnaps. It stores the year/month/day + // index associated with this file int curr_idx = -1; // A snapshot fits if // - type=NumSnaps: the number of stored snaps is less than the max allowed per file. - // - otherwise: the snapshot month/year index match the one currently stored in the file + // - otherwise: the snapshot year/month/day index match the one currently stored in the file // or the file has no snapshot stored yet - bool snapshot_fits (const util::TimeStamp& t) { - const auto& idx = type==Monthly ? t.get_month() : t.get_year(); + bool snapshot_fits (const util::TimeStamp& t) const { switch (type) { - case Yearly: - case Monthly: - return curr_idx==-1 or curr_idx==idx; + case Yearly: [[fallthrough]]; + case Monthly: [[fallthrough]]; + case Daily: + return curr_idx==-1 or curr_idx==t.get_date()[static_cast(type)]; case NumSnaps: return num_snapshots_in_file(type)]; [[fallthrough]]; + case NumSnaps: + ++num_snapshots_in_file; break; default: EKAT_ERROR_MSG ("Error! Unrecognized/unsupported file storage type.\n"); } @@ -86,9 +95,8 @@ struct IOFileSpecs { // If positive, flush the output file every these many snapshots int flush_frequency = std::numeric_limits::max(); - // bool file_is_full () const { return num_snapshots_in_file>=max_snapshots_in_file; } bool file_needs_flush () const { - return storage.num_snapshots_in_file%flush_frequency==0; + return storage.num_snapshots_in_file>0 and storage.num_snapshots_in_file%flush_frequency==0; } // Whether it is a model output, model restart, or history restart file diff --git a/components/eamxx/src/share/io/scream_io_utils.cpp b/components/eamxx/src/share/io/scream_io_utils.cpp index 9318728d657a..cacb153b929e 100644 --- a/components/eamxx/src/share/io/scream_io_utils.cpp +++ b/components/eamxx/src/share/io/scream_io_utils.cpp @@ -1,6 +1,7 @@ #include "share/io/scream_io_utils.hpp" #include "share/io/scream_scorpio_interface.hpp" +#include "share/grid/library_grids_manager.hpp" #include "share/util/scream_utils.hpp" #include "share/scream_config.hpp" @@ -117,4 +118,99 @@ util::TimeStamp read_timestamp (const std::string& filename, return ts; } +std::shared_ptr +create_diagnostic (const std::string& diag_field_name, + const std::shared_ptr& grid) +{ + // Note: use grouping (the (..) syntax), so you can later query the content + // of each group in the matches output var! + // Note: use raw string syntax R"()" to avoid having to escape the \ character + // Note: the number for field_at_p/h can match positive integer/floating-point numbers + std::regex field_at_l (R"(([A-Za-z0-9_]+)_at_(lev_(\d+)|model_(top|bot))$)"); + std::regex field_at_p (R"(([A-Za-z0-9_]+)_at_(\d+(\.\d+)?)(hPa|mb|Pa)$)"); + std::regex field_at_h (R"(([A-Za-z0-9_]+)_at_(\d+(\.\d+)?)(m)_above_(sealevel|surface)$)"); + std::regex surf_mass_flux ("precip_(liq|ice|total)_surf_mass_flux$"); + std::regex water_path ("(Ice|Liq|Rain|Rime|Vap)WaterPath$"); + std::regex number_path ("(Ice|Liq|Rain)NumberPath$"); + std::regex aerocom_cld ("AeroComCld(Top|Bot)$"); + std::regex vap_flux ("(Meridional|Zonal)VapFlux$"); + std::regex backtend ("([A-Za-z0-9_]+)_atm_backtend$"); + std::regex pot_temp ("(Liq)?PotentialTemperature$"); + std::regex vert_layer ("(z|geopotential|height)_(mid|int)$"); + std::regex horiz_avg ("([A-Za-z0-9_]+)_horiz_avg$"); + + std::string diag_name; + std::smatch matches; + ekat::ParameterList params(diag_field_name); + + if (std::regex_search(diag_field_name,matches,field_at_l)) { + params.set("field_name",matches[1].str()); + params.set("grid_name",grid->name()); + params.set("vertical_location", matches[2].str()); + diag_name = "FieldAtLevel"; + } else if (std::regex_search(diag_field_name,matches,field_at_p)) { + params.set("field_name",matches[1].str()); + params.set("grid_name",grid->name()); + params.set("pressure_value",matches[2].str()); + params.set("pressure_units", matches[4].str()); + diag_name = "FieldAtPressureLevel"; + } else if (std::regex_search(diag_field_name,matches,field_at_h)) { + params.set("field_name",matches[1].str()); + params.set("grid_name",grid->name()); + params.set("height_value",matches[2].str()); + params.set("height_units",matches[4].str()); + params.set("surface_reference", matches[5].str()); + diag_name = "FieldAtHeight"; + } else if (std::regex_search(diag_field_name,matches,surf_mass_flux)) { + diag_name = "precip_surf_mass_flux"; + params.set("precip_type",matches[1].str()); + } else if (std::regex_search(diag_field_name,matches,water_path)) { + diag_name = "WaterPath"; + params.set("Water Kind",matches[1].str()); + } else if (std::regex_search(diag_field_name,matches,number_path)) { + diag_name = "NumberPath"; + params.set("Number Kind",matches[1].str()); + } else if (std::regex_search(diag_field_name,matches,aerocom_cld)) { + diag_name = "AeroComCld"; + params.set("AeroComCld Kind",matches[1].str()); + } else if (std::regex_search(diag_field_name,matches,vap_flux)) { + diag_name = "VaporFlux"; + params.set("Wind Component",matches[1].str()); + } else if (std::regex_search(diag_field_name,matches,backtend)) { + diag_name = "AtmBackTendDiag"; + // Set the grid_name + params.set("grid_name",grid->name()); + params.set("Tendency Name",matches[1].str()); + } else if (std::regex_search(diag_field_name,matches,pot_temp)) { + diag_name = "PotentialTemperature"; + params.set("Temperature Kind", matches[1].str()!="" ? matches[1].str() : std::string("Tot")); + } else if (std::regex_search(diag_field_name,matches,vert_layer)) { + diag_name = "VerticalLayer"; + params.set("diag_name",matches[1].str()); + params.set("vert_location",matches[2].str()); + } else if (diag_field_name=="dz") { + diag_name = "VerticalLayer"; + params.set("diag_name","dz"); + params.set("vert_location","mid"); + } + else if (std::regex_search(diag_field_name,matches,horiz_avg)) { + diag_name = "HorizAvgDiag"; + // Set the grid_name + params.set("grid_name",grid->name()); + params.set("field_name",matches[1].str()); + } + else + { + // No existing special regex matches, so we assume that the diag field name IS the diag name. + diag_name = diag_field_name; + } + + auto comm = grid->get_comm(); + auto diag = AtmosphereDiagnosticFactory::instance().create(diag_name,comm,params); + auto gm = std::make_shared(grid); + diag->set_grids(gm); + + return diag; +} + } // namespace scream diff --git a/components/eamxx/src/share/io/scream_io_utils.hpp b/components/eamxx/src/share/io/scream_io_utils.hpp index 01bc46e1e601..0bea91dc3fac 100644 --- a/components/eamxx/src/share/io/scream_io_utils.hpp +++ b/components/eamxx/src/share/io/scream_io_utils.hpp @@ -3,11 +3,14 @@ #include "scream_io_control.hpp" #include "share/util/scream_time_stamp.hpp" +#include "share/atm_process/atmosphere_diagnostic.hpp" +#include "share/grid/abstract_grid.hpp" #include #include #include +#include namespace scream { @@ -72,28 +75,6 @@ std::string find_filename_in_rpointer ( const OutputAvgType avg_type = OutputAvgType::Instant, const IOControl& control = {}); -struct LongNames { - - std::string get_longname (const std::string& name) { - if (name_2_longname.count(name)>0) { - return name_2_longname.at(name); - } else { - // TODO: Do we want to print a Warning message? I'm not sure if its needed. - return name; - } - } - - // Create map of longnames, can be added to as developers see fit. - std::map name_2_longname = { - {"lev","hybrid level at midpoints (1000*(A+B))"}, - {"hyai","hybrid A coefficient at layer interfaces"}, - {"hybi","hybrid B coefficient at layer interfaces"}, - {"hyam","hybrid A coefficient at layer midpoints"}, - {"hybm","hybrid B coefficient at layer midpoints"} - }; - -}; - // Shortcut to write/read to/from YYYYMMDD/HHMMSS attributes in the NC file void write_timestamp (const std::string& filename, const std::string& ts_name, const util::TimeStamp& ts, const bool write_nsteps = false); @@ -101,5 +82,11 @@ util::TimeStamp read_timestamp (const std::string& filename, const std::string& ts_name, const bool read_nsteps = false); +// Create a diagnostic from a string representation of it. +// E.g., create the diag to compute fieldX_at_500hPa. +std::shared_ptr +create_diagnostic (const std::string& diag_name, + const std::shared_ptr& grid); + } // namespace scream #endif // SCREAM_IO_UTILS_HPP diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index 4f35827ba3f4..050a55936b4a 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -24,45 +24,44 @@ OutputManager:: } void OutputManager:: -setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, - const std::shared_ptr& field_mgr, - const std::shared_ptr& grids_mgr, - const util::TimeStamp& run_t0, - const util::TimeStamp& case_t0, - const bool is_model_restart_output) -{ - using map_t = std::map>; - map_t fms; - fms[field_mgr->get_grid()->name()] = field_mgr; - setup(io_comm,params,fms,grids_mgr,run_t0,case_t0,is_model_restart_output); -} - -void OutputManager:: -setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, - const std::map>& field_mgrs, - const std::shared_ptr& grids_mgr, - const util::TimeStamp& run_t0, - const util::TimeStamp& case_t0, - const bool is_model_restart_output) +initialize(const ekat::Comm& io_comm, const ekat::ParameterList& params, + const util::TimeStamp& run_t0, const util::TimeStamp& case_t0, + const bool is_model_restart_output, const RunType run_type) { // Sanity checks EKAT_REQUIRE_MSG (run_t0.is_valid(), + "Error! Invalid run_t0 timestamp: " + run_t0.to_string() + "\n"); + EKAT_REQUIRE_MSG (case_t0.is_valid(), "Error! Invalid case_t0 timestamp: " + case_t0.to_string() + "\n"); - EKAT_REQUIRE_MSG (run_t0.is_valid(), - "Error! Invalid run_t0 timestamp: " + case_t0.to_string() + "\n"); EKAT_REQUIRE_MSG (case_t0<=run_t0, "Error! The case_t0 timestamp must precede run_t0.\n" " run_t0 : " + run_t0.to_string() + "\n" " case_t0: " + case_t0.to_string() + "\n"); m_io_comm = io_comm; + m_params = params; m_run_t0 = run_t0; m_case_t0 = case_t0; - m_is_restarted_run = (case_t0& field_mgr, + const std::shared_ptr& grids_mgr) +{ + using map_t = std::map>; + map_t fms; + fms[field_mgr->get_grid()->name()] = field_mgr; + setup(fms,grids_mgr); +} +void OutputManager:: +setup (const std::map>& field_mgrs, + const std::shared_ptr& grids_mgr) +{ // Read input parameters and setup internal data - set_params(params,field_mgrs); + setup_internals(field_mgrs); // Here, store if PG2 fields will be present in output streams. // Will be useful if multiple grids are defined (see below). @@ -162,7 +161,7 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, // Note: the user might decide *not* to restart the output, so give the option // of disabling the restart. Also, the user might want to change the // filename_prefix, so allow to specify a different filename_prefix for the restart file. - if (m_is_restarted_run and not m_is_model_restart_output) { + if (m_run_type==RunType::Restart and not m_is_model_restart_output) { // Allow to skip history restart, or to specify a filename_prefix for the restart file // that is different from the filename_prefix of the current output. auto& restart_pl = m_params.sublist("Restart"); @@ -230,17 +229,17 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, const auto& last_output_filename = get_attribute(rhist_file,"GLOBAL","last_output_filename"); m_resume_output_file = last_output_filename!="" and not restart_pl.get("force_new_file",false); if (m_resume_output_file) { - int num_snaps = scorpio::get_attribute(rhist_file,"GLOBAL","last_output_file_num_snaps"); - - m_output_file_specs.filename = last_output_filename; - m_output_file_specs.is_open = true; - m_output_file_specs.storage.num_snapshots_in_file = num_snaps; + m_output_file_specs.storage.num_snapshots_in_file = scorpio::get_attribute(rhist_file,"GLOBAL","last_output_file_num_snaps"); if (m_output_file_specs.storage.snapshot_fits(m_output_control.next_write_ts)) { // The setup_file call will not register any new variable (the file is in Append mode, // so all dims/vars must already be in the file). However, it will register decompositions, // since those are a property of the run, not of the file. + m_output_file_specs.filename = last_output_filename; + m_output_file_specs.is_open = true; setup_file(m_output_file_specs,m_output_control); + } else { + m_output_file_specs.close(); } } scorpio::release_file(rhist_file); @@ -253,7 +252,7 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, m_time_bnds.resize(2); m_time_bnds[0] = m_run_t0.days_from(m_case_t0); } else if (m_output_control.output_enabled() and - m_run_t0==m_case_t0 and + m_run_type==RunType::Initial and not m_is_model_restart_output and not m_params.sublist("output_control").get("skip_t0_output",false)) // This will be true for ERS/ERP tests { @@ -287,17 +286,31 @@ void OutputManager::init_timestep (const util::TimeStamp& start_of_step, const R return; } - // Check if the end of this timestep will correspond to an output step. If not, there's nothing to do - const auto& end_of_step = start_of_step+dt; + const bool is_first_step = m_output_control.dt==0 and dt>0; + + // Make sure dt is in the control + m_output_control.set_dt(dt); + + if (m_run_type==RunType::Initial and is_first_step and m_avg_type==OutputAvgType::Instant and + m_output_file_specs.storage.type!=NumSnaps and m_output_control.frequency_units=="nsteps") { + // This is the 1st step of the whole run, and a very sneaky corner case. Bear with me. + // When we call run, we also compute next_write_ts. Then, we use next_write_ts to see if the + // next output step will fit in the currently open file, and, if not, close it right away. + // For a storage type!=NumSnaps, we need to have a valid timestamp for next_write_ts, which + // for freq=nsteps requires to know dt. But at t=case_t0, we did NOT have dt, which means we + // computed next_write_ts=last_write_ts (in terms of date:time, the num_steps is correct). + // This means that at that time we deemed that the next_write_ts definitely fit in the same + // file as last_write_ts (date/time are the same!), which may or may not be true for non NumSnaps + // storage. To fix this, we recompute next_write_ts here, and close the file if it doesn't. + m_output_control.compute_next_write_ts(); + close_or_flush_if_needed (m_output_file_specs,m_output_control); + } - // Note: a full checkpoint not only writes globals in the restart file, but also all the history variables. - // Since we *always* write a history restart file, we can have a non-full checkpoint, if the average - // type is Instant and/or the frequency is every step. A non-full checkpoint will simply write some - // global attribute, such as the time of last write. - const bool is_output_step = m_output_control.is_write_step(end_of_step) || end_of_step==m_case_t0; - const bool is_checkpoint_step = m_checkpoint_control.is_write_step(end_of_step); - const bool has_checkpoint_data = m_avg_type!=OutputAvgType::Instant; - if (not is_output_step and not (is_checkpoint_step and has_checkpoint_data) ) { + // Note: we need to "init" the timestep if we are going to do something this step, which means we either + // have INST output and it's a write step, or we have AVG output. + const auto end_of_step = start_of_step+dt; + const bool is_output_step = m_output_control.is_write_step(end_of_step) || end_of_step==m_case_t0; + if (not is_output_step and m_avg_type==OutputAvgType::Instant) { return; } @@ -328,10 +341,6 @@ void OutputManager::run(const util::TimeStamp& timestamp) "The most likely cause is an output frequency that is faster than the atm timestep.\n" "Try to increase 'Frequency' and/or 'frequency_units' in your output yaml file.\n"); - // Update counters - ++m_output_control.nsamples_since_last_write; - ++m_checkpoint_control.nsamples_since_last_write; - if (m_atm_logger) { m_atm_logger->debug("[OutputManager::run] filename_prefix: " + m_filename_prefix + "\n"); } @@ -361,6 +370,14 @@ void OutputManager::run(const util::TimeStamp& timestamp) const bool is_full_checkpoint_step = is_checkpoint_step && has_checkpoint_data && not is_output_step; const bool is_write_step = is_output_step || is_checkpoint_step; + // Update counters + ++m_output_control.nsamples_since_last_write; + if (not is_t0_output) { + // In case REST_OPT=nsteps, don't count t0 output as one of those steps + // NOTE: for m_output_control, it doesn't matter, since it'll be reset to 0 before we return + ++m_checkpoint_control.nsamples_since_last_write; + } + // Create and setup output/checkpoint file(s), if necessary start_timer(timer_root+"::get_new_file"); auto setup_output_file = [&](IOControl& control, IOFileSpecs& filespecs) { @@ -381,15 +398,11 @@ void OutputManager::run(const util::TimeStamp& timestamp) snapshot_start = m_case_t0; snapshot_start += m_time_bnds[0]; } - if (filespecs.is_open and not filespecs.storage.snapshot_fits(snapshot_start)) { - release_file(filespecs.filename); - filespecs.close(); - } // Check if we need to open a new file if (not filespecs.is_open) { filespecs.filename = compute_filename (filespecs,timestamp); - // Register all dims/vars, write geometry data (e.g. lat/lon/hyam/hybm) + // Register all dims/vars, write geometry data (e.g. lat/lon/hyam/hybm/hyai/hybi) setup_file(filespecs,control); } @@ -483,10 +496,7 @@ void OutputManager::run(const util::TimeStamp& timestamp) write_timestamp (filespecs.filename,"last_write",m_output_control.last_write_ts,true); scorpio::set_attribute (filespecs.filename,"GLOBAL","last_output_filename",m_output_file_specs.filename); scorpio::set_attribute (filespecs.filename,"GLOBAL","num_snapshots_since_last_write",m_output_control.nsamples_since_last_write); - - int nsnaps = m_output_file_specs.is_open - ? scorpio::get_dimlen(m_output_file_specs.filename,"time") : 0; - scorpio::set_attribute (filespecs.filename,"GLOBAL","last_output_file_num_snaps",nsnaps); + scorpio::set_attribute (filespecs.filename,"GLOBAL","last_output_file_num_snaps",m_output_file_specs.storage.num_snapshots_in_file); } // Write these in both output and rhist file. The former, b/c we need these info when we postprocess // output, and the latter b/c we want to make sure these params don't change across restarts @@ -534,10 +544,7 @@ void OutputManager::run(const util::TimeStamp& timestamp) scorpio::write_var(filespecs.filename, "time_bnds", m_time_bnds.data()); } - // Check if we need to flush the output file - if (filespecs.file_needs_flush()) { - flush_file (filespecs.filename); - } + close_or_flush_if_needed(filespecs,control); }; start_timer(timer_root+"::update_snapshot_tally"); @@ -549,6 +556,11 @@ void OutputManager::run(const util::TimeStamp& timestamp) } if (is_checkpoint_step) { write_global_data(m_checkpoint_control,m_checkpoint_file_specs); + + // Always flush output during checkpoints (assuming we opened it already) + if (m_output_file_specs.is_open) { + scorpio::flush_file (m_output_file_specs.filename); + } } stop_timer(timer_root+"::update_snapshot_tally"); if (is_output_step && m_time_bnds.size()>0) { @@ -622,28 +634,25 @@ compute_filename (const IOFileSpecs& file_specs, auto ts = (m_avg_type==OutputAvgType::Instant || file_specs.ftype==FileType::HistoryRestart) ? timestamp : control.last_write_ts; + int ts_string_len = 0; switch (file_specs.storage.type) { - case NumSnaps: - filename += "." + ts.to_string(); break; - case Yearly: - filename += "." + std::to_string(ts.get_year()); break; - case Monthly: - filename += "." + std::to_string(ts.get_year()) + "-" + std::to_string(ts.get_month()); break; + case Yearly: ts_string_len = 4; break; // YYYY + case Monthly: ts_string_len = 7; break; // YYYY-MM + case Daily: ts_string_len = 10; break; // YYYY-MM-DD + case NumSnaps: ts_string_len = 16; break; // YYYY-MM-DD-XXXXX default: EKAT_ERROR_MSG ("Error! Unrecognized/unsupported file storage type.\n"); } + filename += "." + ts.to_string().substr(0,ts_string_len); return filename + ".nc"; } void OutputManager:: -set_params (const ekat::ParameterList& params, - const std::map>& field_mgrs) +setup_internals (const std::map>& field_mgrs) { using vos_t = std::vector; - m_params = params; - if (m_is_model_restart_output) { // We build some restart parameters internally m_avg_type = OutputAvgType::Instant; @@ -690,6 +699,8 @@ set_params (const ekat::ParameterList& params, storage.type = Yearly; } else if (storage_type=="one_month") { storage.type = Monthly; + } else if (storage_type=="one_day") { + storage.type = Daily; } else { EKAT_ERROR_MSG ("Error! Unrecognized/unsupported file storage type.\n"); } @@ -710,7 +721,7 @@ set_params (const ekat::ParameterList& params, EKAT_REQUIRE_MSG(m_params.isSublist("output_control"), "Error! The output control YAML file for " + m_filename_prefix + " is missing the sublist 'output_control'"); auto& out_control_pl = m_params.sublist("output_control"); - m_output_control.frequency_units = out_control_pl.get("frequency_units"); + m_output_control.set_frequency_units(out_control_pl.get("frequency_units")); // In case output is disabled, no point in doing anything else if (m_output_control.frequency_units=="none" || m_output_control.frequency_units=="never") { @@ -724,7 +735,6 @@ set_params (const ekat::ParameterList& params, // 1. use_case_as_start_reference: TRUE - implies we want to calculate frequency from the beginning of the whole simulation, even if this is a restarted run. // 2. use_case_as_start_reference: FALSE - implies we want to base the frequency of output on when this particular simulation started. // Note, (2) is needed for restarts since the restart frequency in CIME assumes a reference of when this run began. - // NOTE: m_is_restarted_run and not m_is_model_restart_output, these timestamps will be corrected once we open the hist restart file const bool use_case_as_ref = out_control_pl.get("use_case_as_start_reference",!m_is_model_restart_output); const bool perform_history_restart = m_params.sublist("Restart").get("Perform Restart",true); const auto& start_ref = use_case_as_ref and perform_history_restart ? m_case_t0 : m_run_t0; @@ -737,7 +747,7 @@ set_params (const ekat::ParameterList& params, if (m_params.isSublist("Checkpoint Control")) { auto& pl = m_params.sublist("Checkpoint Control"); - m_checkpoint_control.frequency_units = pl.get("frequency_units"); + m_checkpoint_control.set_frequency_units(pl.get("frequency_units")); if (m_checkpoint_control.output_enabled()) { m_checkpoint_control.frequency = pl.get("Frequency"); @@ -891,7 +901,18 @@ void OutputManager::set_file_header(const IOFileSpecs& file_specs) set_str_att("Conventions","CF-1.8"); set_str_att("product",e2str(file_specs.ftype)); } -/*===============================================================================================*/ +void OutputManager:: +close_or_flush_if_needed ( IOFileSpecs& file_specs, + const IOControl& control) const +{ + if (not file_specs.storage.snapshot_fits(control.next_write_ts)) { + scorpio::release_file(file_specs.filename); + file_specs.close(); + } else if (file_specs.file_needs_flush()) { + scorpio::flush_file (file_specs.filename); + } +} + void OutputManager:: push_to_logger() { @@ -903,13 +924,18 @@ push_to_logger() return y; }; + auto rt_to_string = [](RunType rt) { + std::string s = rt==RunType::Initial ? "Initial" : "Restart"; + return s; + }; + m_atm_logger->info("[EAMxx::output_manager] - New Output stream"); m_atm_logger->info(" Filename prefix: " + m_filename_prefix); m_atm_logger->info(" Run t0: " + m_run_t0.to_string()); m_atm_logger->info(" Case t0: " + m_case_t0.to_string()); m_atm_logger->info(" Reference t0: " + m_output_control.last_write_ts.to_string()); m_atm_logger->info(" Is Restart File ?: " + bool_to_string(m_is_model_restart_output)); - m_atm_logger->info(" Is Restarted Run ?: " + bool_to_string(m_is_restarted_run)); + m_atm_logger->info(" Run type : " + rt_to_string(m_run_type)); m_atm_logger->info(" Averaging Type: " + e2str(m_avg_type)); m_atm_logger->info(" Output Frequency: " + std::to_string(m_output_control.frequency) + " " + m_output_control.frequency_units); switch (m_output_file_specs.storage.type) { diff --git a/components/eamxx/src/share/io/scream_output_manager.hpp b/components/eamxx/src/share/io/scream_output_manager.hpp index 4d7486380cfe..1e4f2dba5698 100644 --- a/components/eamxx/src/share/io/scream_output_manager.hpp +++ b/components/eamxx/src/share/io/scream_output_manager.hpp @@ -67,47 +67,46 @@ class OutputManager using globals_map_t = std::map; // Constructor(s) & Destructor - OutputManager () = default; + OutputManager() = default; virtual ~OutputManager (); - // Set up the manager, creating all output streams. Inputs: + // Initialize manager by storing class members that only depend on runtime parameters. + // Inputs: // - params: the parameter list with file/fields info, as well as method of output options - // - field_mgr/field_mgrs: field manager(s) storing fields to be outputed - // - grids_mgr: grid manager to create remapping from field managers grids onto the IO grid. - // This is needed, e.g., when outputing SEGrid fields without duplicating dofs. // - run_t0: the timestamp of the start of the current simulation // - case_t0: the timestamp of the start of the overall simulation (precedes run_r0 for // a restarted simulation. Restart logic is triggered *only* if case_t0& field_mgr, - const std::shared_ptr& grids_mgr, - const util::TimeStamp& run_t0, - const util::TimeStamp& case_t0, - const bool is_model_restart_output); - void setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, - const std::shared_ptr& field_mgr, - const std::shared_ptr& grids_mgr, - const util::TimeStamp& run_t0, - const bool is_model_restart_output) { - setup (io_comm,params,field_mgr,grids_mgr,run_t0,run_t0,is_model_restart_output); + void initialize (const ekat::Comm& io_comm, const ekat::ParameterList& params, + const util::TimeStamp& run_t0, const util::TimeStamp& case_t0, + const bool is_model_restart_output, const RunType run_type); + + // This overloads are to make certain unit tests easier + void initialize (const ekat::Comm& io_comm, const ekat::ParameterList& params, + const util::TimeStamp& run_t0, const util::TimeStamp& case_t0, + const bool is_model_restart_output) + { + auto run_type = case_t0>& field_mgrs, - const std::shared_ptr& grids_mgr, - const util::TimeStamp& run_t0, - const util::TimeStamp& case_t0, - const bool is_model_restart_output); - - void setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, - const std::map>& field_mgrs, - const std::shared_ptr& grids_mgr, - const util::TimeStamp& run_t0, - const bool is_model_restart_output) { - setup (io_comm,params,field_mgrs,grids_mgr,run_t0,run_t0,is_model_restart_output); + void initialize (const ekat::Comm& io_comm, const ekat::ParameterList& params, + const util::TimeStamp& run_t0, + const bool is_model_restart_output) + { + initialize(io_comm, params, run_t0, run_t0, is_model_restart_output, RunType::Initial); } + // Setup manager by creating the internal output streams using grids/field data + // Inputs: + // - field_mgr/field_mgrs: field manager(s) storing fields to be outputed + // - grids_mgr: grid manager to create remapping from field managers grids onto the IO grid. + // This is needed, e.g., when outputing SEGrid fields without duplicating dofs. + void setup (const std::shared_ptr& field_mgr, + const std::shared_ptr& grids_mgr); + + void setup (const std::map>& field_mgrs, + const std::shared_ptr& grids_mgr); + void set_logger(const std::shared_ptr& atm_logger) { m_atm_logger = atm_logger; } @@ -118,6 +117,12 @@ class OutputManager void finalize(); long long res_dep_memory_footprint () const; + + bool is_restart () const { return m_is_model_restart_output; } + + // For debug and testing purposes + const IOControl& output_control () const { return m_output_control; } + const IOFileSpecs& output_file_specs () const { return m_output_file_specs; } protected: std::string compute_filename (const IOFileSpecs& file_specs, @@ -125,13 +130,17 @@ class OutputManager void set_file_header(const IOFileSpecs& file_specs); - // Craft the restart parameter list - void set_params (const ekat::ParameterList& params, - const std::map>& field_mgrs); + // Set internal class variables and processes the field_mgrs for restart fields + // to add to the parameter list for a model restart managers. + void setup_internals (const std::map>& field_mgrs); void setup_file ( IOFileSpecs& filespecs, const IOControl& control); + // If a file can be closed (next snap won't fit) or needs flushing, do so + void close_or_flush_if_needed ( IOFileSpecs& file_specs, + const IOControl& control) const; + // Manage logging of info to atm.log void push_to_logger(); @@ -169,7 +178,7 @@ class OutputManager // Whether this run is the restart of a previous run, in which case // we might have to load an output checkpoint file (depending on avg type) - bool m_is_restarted_run; + RunType m_run_type; // Whether a restarted run can resume filling previous run output file (if not full) bool m_resume_output_file = false; diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index 8d2f64994ddd..c19740dbcc11 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -141,6 +141,7 @@ int nctype (const std::string& type) { } else { EKAT_ERROR_MSG ("Error! Unrecognized/unsupported data type '" + type + "'.\n"); } + return -1; } template diff --git a/components/eamxx/src/share/io/tests/CMakeLists.txt b/components/eamxx/src/share/io/tests/CMakeLists.txt index 8c819f7896cc..89d92075723d 100644 --- a/components/eamxx/src/share/io/tests/CMakeLists.txt +++ b/components/eamxx/src/share/io/tests/CMakeLists.txt @@ -17,6 +17,12 @@ CreateUnitTest(io_utils "io_utils.cpp" PROPERTIES RESOURCE_LOCK rpointer_file ) +# Test creation of diagnostic from diag_field_name +CreateUnitTest(create_diag "create_diag.cpp" + LIBS diagnostics scream_io + LABELS io diagnostics +) + ## Test basic output (no packs, no diags, all avg types, all freq units) CreateUnitTest(io_basic "io_basic.cpp" LIBS scream_io LABELS io diff --git a/components/eamxx/src/share/io/tests/create_diag.cpp b/components/eamxx/src/share/io/tests/create_diag.cpp new file mode 100644 index 000000000000..bc5091759719 --- /dev/null +++ b/components/eamxx/src/share/io/tests/create_diag.cpp @@ -0,0 +1,166 @@ +#include "catch2/catch.hpp" + +#include "diagnostics/register_diagnostics.hpp" + +#include "share/io/scream_io_utils.hpp" +#include "share/grid/point_grid.hpp" + +namespace scream { + +TEST_CASE("create_diag") +{ + ekat::Comm comm(MPI_COMM_WORLD); + + register_diagnostics(); + + // Create a grid + const int ncols = 3*comm.size(); + const int nlevs = 10; + auto grid = create_point_grid("Physics",ncols,nlevs,comm); + + SECTION ("field_at") { + // FieldAtLevel + auto d1 = create_diagnostic("BlaH_123_at_model_top",grid); + REQUIRE (std::dynamic_pointer_cast(d1)!=nullptr); + auto d2 = create_diagnostic("BlaH_123_at_model_bot",grid); + REQUIRE (std::dynamic_pointer_cast(d2)!=nullptr); + auto d3 = create_diagnostic("BlaH_123_at_lev_10",grid); + REQUIRE (std::dynamic_pointer_cast(d3)!=nullptr); + + REQUIRE_THROWS(create_diagnostic("BlaH_123_at_modeltop",grid)); // misspelled + + // FieldAtPressureLevel + auto d4 = create_diagnostic("BlaH_123_at_10mb",grid); + REQUIRE (std::dynamic_pointer_cast(d4)!=nullptr); + auto d5 = create_diagnostic("BlaH_123_at_10hPa",grid); + REQUIRE (std::dynamic_pointer_cast(d5)!=nullptr); + auto d6 = create_diagnostic("BlaH_123_at_10Pa",grid); + REQUIRE (std::dynamic_pointer_cast(d6)!=nullptr); + + REQUIRE_THROWS(create_diagnostic("BlaH_123_at_400KPa",grid)); // invalid units + + // FieldAtHeight + auto d7 = create_diagnostic("BlaH_123_at_10m_above_sealevel",grid); + REQUIRE (std::dynamic_pointer_cast(d7)!=nullptr); + auto d8 = create_diagnostic("BlaH_123_at_10m_above_surface",grid); + REQUIRE (std::dynamic_pointer_cast(d8)!=nullptr); + + REQUIRE_THROWS(create_diagnostic("BlaH_123_at_10.5m",grid)); // missing _above_X + REQUIRE_THROWS(create_diagnostic("BlaH_123_at_1km_above_sealevel",grid)); // invalid units + REQUIRE_THROWS(create_diagnostic("BlaH_123_at_1m_above_the_surface",grid)); // invalid reference + } + + SECTION ("precip_mass_flux") { + auto d1 = create_diagnostic("precip_liq_surf_mass_flux",grid); + REQUIRE (std::dynamic_pointer_cast(d1)!=nullptr); + REQUIRE (d1->get_params().get("precip_type")=="liq"); + + auto d2 = create_diagnostic("precip_ice_surf_mass_flux",grid); + REQUIRE (std::dynamic_pointer_cast(d2)!=nullptr); + REQUIRE (d2->get_params().get("precip_type")=="ice"); + + auto d3 = create_diagnostic("precip_total_surf_mass_flux",grid); + REQUIRE (std::dynamic_pointer_cast(d3)!=nullptr); + REQUIRE (d3->get_params().get("precip_type")=="total"); + } + + SECTION ("water_and_number_path") { + auto d1 = create_diagnostic("LiqWaterPath",grid); + REQUIRE (std::dynamic_pointer_cast(d1)!=nullptr); + REQUIRE (d1->get_params().get("Water Kind")=="Liq"); + + auto d2 = create_diagnostic("IceWaterPath",grid); + REQUIRE (std::dynamic_pointer_cast(d2)!=nullptr); + REQUIRE (d2->get_params().get("Water Kind")=="Ice"); + + auto d3 = create_diagnostic("RainWaterPath",grid); + REQUIRE (std::dynamic_pointer_cast(d3)!=nullptr); + REQUIRE (d3->get_params().get("Water Kind")=="Rain"); + + auto d4 = create_diagnostic("RimeWaterPath",grid); + REQUIRE (std::dynamic_pointer_cast(d4)!=nullptr); + REQUIRE (d4->get_params().get("Water Kind")=="Rime"); + + auto d5 = create_diagnostic("VapWaterPath",grid); + REQUIRE (std::dynamic_pointer_cast(d5)!=nullptr); + REQUIRE (d5->get_params().get("Water Kind")=="Vap"); + + auto d6 = create_diagnostic("LiqNumberPath",grid); + REQUIRE (std::dynamic_pointer_cast(d6)!=nullptr); + REQUIRE (d6->get_params().get("Number Kind")=="Liq"); + + auto d7 = create_diagnostic("IceNumberPath",grid); + REQUIRE (std::dynamic_pointer_cast(d7)!=nullptr); + REQUIRE (d7->get_params().get("Number Kind")=="Ice"); + + auto d8 = create_diagnostic("RainNumberPath",grid); + REQUIRE (std::dynamic_pointer_cast(d8)!=nullptr); + REQUIRE (d8->get_params().get("Number Kind")=="Rain"); + } + + SECTION ("aerocom_cld") { + auto d1 = create_diagnostic("AeroComCldTop",grid); + REQUIRE (std::dynamic_pointer_cast(d1)!=nullptr); + REQUIRE (d1->get_params().get("AeroComCld Kind")=="Top"); + + auto d2 = create_diagnostic("AeroComCldBot",grid); + REQUIRE (std::dynamic_pointer_cast(d2)!=nullptr); + REQUIRE (d2->get_params().get("AeroComCld Kind")=="Bot"); + } + + SECTION ("vapor_flux") { + auto d1 = create_diagnostic("MeridionalVapFlux",grid); + REQUIRE (std::dynamic_pointer_cast(d1)!=nullptr); + REQUIRE (d1->get_params().get("Wind Component")=="Meridional"); + + auto d2 = create_diagnostic("ZonalVapFlux",grid); + REQUIRE (std::dynamic_pointer_cast(d2)!=nullptr); + REQUIRE (d2->get_params().get("Wind Component")=="Zonal"); + } + + SECTION ("atm_tend") { + auto d1 = create_diagnostic("BlaH_123_atm_backtend",grid); + REQUIRE (std::dynamic_pointer_cast(d1)!=nullptr); + REQUIRE (d1->get_params().get("Tendency Name")=="BlaH_123"); + } + + SECTION ("pot_temp") { + auto d1 = create_diagnostic("LiqPotentialTemperature",grid); + REQUIRE (std::dynamic_pointer_cast(d1)!=nullptr); + REQUIRE (d1->get_params().get("Temperature Kind")=="Liq"); + + auto d2 = create_diagnostic("PotentialTemperature",grid); + REQUIRE (std::dynamic_pointer_cast(d2)!=nullptr); + REQUIRE (d2->get_params().get("Temperature Kind")=="Tot"); + } + + SECTION ("vert_layer") { + auto d1 = create_diagnostic("z_mid",grid); + REQUIRE (std::dynamic_pointer_cast(d1)!=nullptr); + REQUIRE (d1->get_params().get("vert_location")=="mid"); + auto d2 = create_diagnostic("z_int",grid); + REQUIRE (std::dynamic_pointer_cast(d2)!=nullptr); + REQUIRE (d2->get_params().get("vert_location")=="int"); + + auto d3 = create_diagnostic("height_mid",grid); + REQUIRE (std::dynamic_pointer_cast(d3)!=nullptr); + REQUIRE (d3->get_params().get("vert_location")=="mid"); + auto d4 = create_diagnostic("height_int",grid); + REQUIRE (std::dynamic_pointer_cast(d4)!=nullptr); + REQUIRE (d4->get_params().get("vert_location")=="int"); + + auto d5 = create_diagnostic("geopotential_mid",grid); + REQUIRE (std::dynamic_pointer_cast(d5)!=nullptr); + REQUIRE (d5->get_params().get("vert_location")=="mid"); + auto d6 = create_diagnostic("geopotential_int",grid); + REQUIRE (std::dynamic_pointer_cast(d6)!=nullptr); + REQUIRE (d6->get_params().get("vert_location")=="int"); + + auto d7 = create_diagnostic("dz",grid); + REQUIRE (std::dynamic_pointer_cast(d7)!=nullptr); + REQUIRE (d7->get_params().get("vert_location")=="mid"); + } + +} + +} // namespace scream diff --git a/components/eamxx/src/share/io/tests/io_basic.cpp b/components/eamxx/src/share/io/tests/io_basic.cpp index 033705a4dd4c..fd745d3e2bf5 100644 --- a/components/eamxx/src/share/io/tests/io_basic.cpp +++ b/components/eamxx/src/share/io/tests/io_basic.cpp @@ -87,7 +87,7 @@ get_fm (const std::shared_ptr& grid, // - Uniform_int_distribution returns an int, and the randomize // util checks that return type matches the Field data type. // So wrap the int pdf in a lambda, that does the cast. - std::mt19937_64 engine(seed); + std::mt19937_64 engine(seed); auto my_pdf = [&](std::mt19937_64& engine) -> Real { std::uniform_int_distribution pdf (0,100); Real v = pdf(engine); @@ -105,7 +105,7 @@ get_fm (const std::shared_ptr& grid, }; auto fm = std::make_shared(grid); - + const auto units = ekat::units::Units::nondimensional(); int count=0; using stratts_t = std::map; @@ -145,7 +145,6 @@ void write (const std::string& avg_type, const std::string& freq_units, // Create output params ekat::ParameterList om_pl; - om_pl.set("MPI Ranks in Filename",true); om_pl.set("filename_prefix",std::string("io_basic")); om_pl.set("Field Names",fnames); om_pl.set("Averaging Type", avg_type); @@ -154,14 +153,25 @@ void write (const std::string& avg_type, const std::string& freq_units, ctrl_pl.set("Frequency",freq); ctrl_pl.set("save_grid_data",false); + // While setting this is in practice irrelevant (we would close + // the file anyways at the end of the run), we can test that the OM closes + // the file AS SOON as it's full (before calling finalize) + int max_snaps = num_output_steps; + if (avg_type=="INSTANT") { + ++max_snaps; + } + om_pl.set("Max Snapshots Per File", max_snaps); + // Create Output manager OutputManager om; // Attempt to use invalid fp precision string om_pl.set("Floating Point Precision",std::string("triple")); - REQUIRE_THROWS (om.setup(comm,om_pl,fm,gm,t0,t0,false)); + om.initialize(comm,om_pl,t0,false); + REQUIRE_THROWS (om.setup(fm,gm)); om_pl.set("Floating Point Precision",std::string("single")); - om.setup(comm,om_pl,fm,gm,t0,t0,false); + om.initialize(comm,om_pl,t0,false); + om.setup(fm,gm); // Time loop: ensure we always hit 3 output steps const int nsteps = num_output_steps*freq; @@ -181,6 +191,10 @@ void write (const std::string& avg_type, const std::string& freq_units, om.run (t); } + // Check that the file was closed, since we reached full capacity + const auto& file_specs = om.output_file_specs(); + REQUIRE (not file_specs.is_open); + // Close file and cleanup om.finalize(); } diff --git a/components/eamxx/src/share/io/tests/io_diags.cpp b/components/eamxx/src/share/io/tests/io_diags.cpp index f4658c8fdfa3..5379a0ab89f0 100644 --- a/components/eamxx/src/share/io/tests/io_diags.cpp +++ b/components/eamxx/src/share/io/tests/io_diags.cpp @@ -131,7 +131,7 @@ get_fm (const std::shared_ptr& grid, // - Uniform_int_distribution returns an int, and the randomize // util checks that return type matches the Field data type. // So wrap the int pdf in a lambda, that does the cast. - std::mt19937_64 engine(seed); + std::mt19937_64 engine(seed); auto my_pdf = [&](std::mt19937_64& engine) -> Real { std::uniform_int_distribution pdf (0,100); Real v = pdf(engine); @@ -142,7 +142,7 @@ get_fm (const std::shared_ptr& grid, const int nlevs = grid->get_num_vertical_levels(); auto fm = std::make_shared(grid); - + const auto units = ekat::units::Units::nondimensional(); FL fl ({COL,LEV}, {nlcols,nlevs}); @@ -186,7 +186,6 @@ void write (const int seed, const ekat::Comm& comm) // Create output params ekat::ParameterList om_pl; - om_pl.set("MPI Ranks in Filename",true); om_pl.set("filename_prefix",std::string("io_diags")); om_pl.set("Field Names",fnames); om_pl.set("Averaging Type", std::string("INSTANT")); @@ -197,7 +196,8 @@ void write (const int seed, const ekat::Comm& comm) // Create Output manager OutputManager om; - om.setup(comm,om_pl,fm,gm,t0,t0,false); + om.initialize(comm, om_pl, t0, false); + om.setup(fm,gm); // Run output manager for (auto it : *fm) { diff --git a/components/eamxx/src/share/io/tests/io_filled.cpp b/components/eamxx/src/share/io/tests/io_filled.cpp index 5af906aba64c..b53b5ef45918 100644 --- a/components/eamxx/src/share/io/tests/io_filled.cpp +++ b/components/eamxx/src/share/io/tests/io_filled.cpp @@ -93,7 +93,7 @@ get_fm (const std::shared_ptr& grid, }; auto fm = std::make_shared(grid); - + const auto units = ekat::units::Units::nondimensional(); for (const auto& fl : layouts) { FID fid("f_"+std::to_string(fl.size()),fl,units,grid->name()); @@ -128,7 +128,6 @@ void write (const std::string& avg_type, const std::string& freq_units, // Create output params ekat::ParameterList om_pl; - om_pl.set("MPI Ranks in Filename",true); om_pl.set("filename_prefix",std::string("io_filled")); om_pl.set("Field Names",fnames); om_pl.set("Averaging Type", avg_type); @@ -142,7 +141,8 @@ void write (const std::string& avg_type, const std::string& freq_units, // Create Output manager OutputManager om; - om.setup(comm,om_pl,fm,gm,t0,t0,false); + om.initialize(comm,om_pl,t0,false); + om.setup(fm,gm); // Time loop: ensure we always hit 3 output steps const int nsteps = num_output_steps*freq; @@ -208,13 +208,13 @@ void read (const std::string& avg_type, const std::string& freq_units, // Hence, at output step N = snap*freq, we should get // avg=INSTANT: output = N if (N%2=0), else Fillvalue // avg=MAX: output = N if (N%2=0), else N-1 - // avg=MIN: output = N + 1, where n is the first timesnap of the Nth output step. + // avg=MIN: output = N + 1, where n is the first timesnap of the Nth output step. // we add + 1 more in cases where (N%2=0) because that means the first snap was filled. // avg=AVERAGE: output = a + M+1 = a + M*(M+1)/M // The last one comes from // a + 2*(1 + 2 +..+M)/M = // a + 2*sum(i)/M = a + 2*(M(M+1)/2)/M, - // where M = freq/2 + ( N%2=0 ? 0 : 1 ), + // where M = freq/2 + ( N%2=0 ? 0 : 1 ), // a = floor(N/freq)*freq + ( N%2=0 ? 0 : -1) for (int n=0; n& grid, // - Uniform_int_distribution returns an int, and the randomize // util checks that return type matches the Field data type. // So wrap the int pdf in a lambda, that does the cast. - std::mt19937_64 engine(seed); + std::mt19937_64 engine(seed); auto my_pdf = [&](std::mt19937_64& engine) -> Real { std::uniform_int_distribution pdf (0,100); Real v = pdf(engine); @@ -83,7 +83,7 @@ get_fm (const std::shared_ptr& grid, }; auto fm = std::make_shared(grid); - + const auto units = ekat::units::Units::nondimensional(); int count=0; for (const auto& fl : layouts) { @@ -119,7 +119,6 @@ void write (const int seed, const ekat::Comm& comm) // Create output params ekat::ParameterList om_pl; - om_pl.set("MPI Ranks in Filename",true); om_pl.set("filename_prefix",std::string("io_monthly")); om_pl.set("Field Names",fnames); om_pl.set("Averaging Type", std::string("Instant")); @@ -132,7 +131,8 @@ void write (const int seed, const ekat::Comm& comm) // Create Output manager OutputManager om; - om.setup(comm,om_pl,fm,gm,t0,t0,false); + om.initialize(comm,om_pl,t0,false); + om.setup(fm,gm); // Time loop: do 11 steps, since we already did Jan output at t0 const int nsteps = 11; @@ -178,11 +178,11 @@ void read (const int seed, const ekat::Comm& comm) // Get filename from timestamp std::string casename = "io_monthly"; auto get_filename = [&](const util::TimeStamp& t) { + auto t_str = t.to_string().substr(0,7); std::string fname = casename + ".INSTANT.nsteps_x1" + ".np" + std::to_string(comm.size()) - + "." + std::to_string(t.get_year()) - + "-" + std::to_string(t.get_month()) + + "." + t_str + ".nc"; return fname; }; diff --git a/components/eamxx/src/share/io/tests/io_packed.cpp b/components/eamxx/src/share/io/tests/io_packed.cpp index 08182b8e6acf..58c819dd19b4 100644 --- a/components/eamxx/src/share/io/tests/io_packed.cpp +++ b/components/eamxx/src/share/io/tests/io_packed.cpp @@ -62,7 +62,7 @@ get_fm (const std::shared_ptr& grid, // - Uniform_int_distribution returns an int, and the randomize // util checks that return type matches the Field data type. // So wrap the int pdf in a lambda, that does the cast. - std::mt19937_64 engine(seed); + std::mt19937_64 engine(seed); auto my_pdf = [&](std::mt19937_64& engine) -> Real { std::uniform_int_distribution pdf (0,100); Real v = pdf(engine); @@ -79,7 +79,7 @@ get_fm (const std::shared_ptr& grid, }; auto fm = std::make_shared(grid); - + const auto units = ekat::units::Units::nondimensional(); for (const auto& fl : layouts) { FID fid("f_"+std::to_string(fl.size()),fl,units,grid->name()); @@ -113,7 +113,6 @@ void write (const int freq, const int seed, const int ps, const ekat::Comm& comm // Create output params ekat::ParameterList om_pl; - om_pl.set("MPI Ranks in Filename",true); om_pl.set("filename_prefix","io_packed_ps"+std::to_string(ps)); om_pl.set("Field Names",fnames); om_pl.set("Averaging Type", std::string("INSTANT")); @@ -124,7 +123,8 @@ void write (const int freq, const int seed, const int ps, const ekat::Comm& comm // Create Output manager OutputManager om; - om.setup(comm,om_pl,fm,gm,t0,t0,false); + om.initialize(comm,om_pl,t0,false); + om.setup(fm,gm); // Run output manager om.init_timestep(t0,0); diff --git a/components/eamxx/src/share/io/tests/io_remap_test.cpp b/components/eamxx/src/share/io/tests/io_remap_test.cpp index 96b2f3e28059..55095ef8a96a 100644 --- a/components/eamxx/src/share/io/tests/io_remap_test.cpp +++ b/components/eamxx/src/share/io/tests/io_remap_test.cpp @@ -74,7 +74,7 @@ TEST_CASE("io_remap_test","io_remap_test") auto field_manager = get_test_fm(grid, false); field_manager->init_fields_time_stamp(t0); print (" -> Test Setup ... done\n",io_comm); - + // Create remap data for both vertical and horizontal remapping // The strategy for remapping will be to map every 2 subsequent // columns to a single column. So we assume that `ncols_src` is @@ -100,7 +100,7 @@ TEST_CASE("io_remap_test","io_remap_test") std::vector p_tgt; for (int ii=0; ii 4 * p_surf = | 0.5*(p_top+p_bot) for |x| < 2 * \ p_bot - (-sign(x)*m + b) otherwise, where m = (p_top+p_bot)/4.0 and b = p_top+p_bot - * - * + * + * * ---- * / \ * / \ @@ -180,7 +180,7 @@ TEST_CASE("io_remap_test","io_remap_test") pi_v(ii,0) = set_pressure(p_top, p_surf(ii), nlevs_src+1, 0); for (int jj=0; jj source data ... \n",io_comm); auto source_remap_control = set_output_params("remap_source",remap_filename,p_ref,false,false); - om_source.setup(io_comm,source_remap_control,field_manager,gm,t0,t0,false); + om_source.initialize(io_comm,source_remap_control,t0,false); + om_source.setup(field_manager,gm); io_comm.barrier(); om_source.init_timestep(t0,dt); om_source.run(t0+dt); @@ -243,7 +244,8 @@ TEST_CASE("io_remap_test","io_remap_test") print (" -> vertical remap ... \n",io_comm); auto vert_remap_control = set_output_params("remap_vertical",remap_filename,p_ref,true,false); - om_vert.setup(io_comm,vert_remap_control,field_manager,gm,t0,t0,false); + om_vert.initialize(io_comm,vert_remap_control,t0,false); + om_vert.setup(field_manager,gm); io_comm.barrier(); om_vert.init_timestep(t0,dt); om_vert.run(t0+dt); @@ -252,7 +254,8 @@ TEST_CASE("io_remap_test","io_remap_test") print (" -> horizontal remap ... \n",io_comm); auto horiz_remap_control = set_output_params("remap_horizontal",remap_filename,p_ref,false,true); - om_horiz.setup(io_comm,horiz_remap_control,field_manager,gm,t0,t0,false); + om_horiz.initialize(io_comm,horiz_remap_control,t0,false); + om_horiz.setup(field_manager,gm); io_comm.barrier(); om_horiz.init_timestep(t0,dt); om_horiz.run(t0+dt); @@ -261,7 +264,8 @@ TEST_CASE("io_remap_test","io_remap_test") print (" -> vertical-horizontal remap ... \n",io_comm); auto vert_horiz_remap_control = set_output_params("remap_vertical_horizontal",remap_filename,p_ref,true,true); - om_vert_horiz.setup(io_comm,vert_horiz_remap_control,field_manager,gm,t0,t0,false); + om_vert_horiz.initialize(io_comm,vert_horiz_remap_control,t0,false); + om_vert_horiz.setup(field_manager,gm); io_comm.barrier(); om_vert_horiz.init_timestep(t0,dt); om_vert_horiz.run(t0+dt); @@ -328,12 +332,12 @@ TEST_CASE("io_remap_test","io_remap_test") const bool ref_masked = (p_ref>pi_v(ii,nlevs_src) || p_refpm_v(ii,nlevs_src-1) || p_jjpi_v(ii,nlevs_src) || p_jjpm_v(ii,nlevs_src-1) || p_jjpi_v(ii,nlevs_src) || p_jj=pi_v(col2,0)) { found = true; Ys_exp += calculate_output(p_ref,col2,0)*(1.0-wgt); @@ -465,7 +469,7 @@ TEST_CASE("io_remap_test","io_remap_test") // also translate to more masking in the horizontal reamapping. So we must check for potential // masking for all variables rather than just the Y_int_at_XPa variable for the horizontal interpolation. // - // NOTE: For scorpio_output.cpp the mask value for vertical remapping is std::numeric_limits::max()/10.0 + // NOTE: For scorpio_output.cpp the mask value for vertical remapping is std::numeric_limits::max()/10.0 const auto& Yf_f_vh = fm_vh->get_field("Y_flat"); const auto& Ys_f_vh = fm_vh->get_field("Y_int_at_"+std::to_string(p_ref)+"Pa"); const auto& Ym_f_vh = fm_vh->get_field("Y_mid"); @@ -504,7 +508,7 @@ TEST_CASE("io_remap_test","io_remap_test") // This point is completely masked out, assign masked value test_int = mask_val; } - REQUIRE(approx(Ym_v_vh(ii,jj), test_mid)); + REQUIRE(approx(Ym_v_vh(ii,jj), test_mid)); REQUIRE(approx(Yi_v_vh(ii,jj), test_int)); for (int cc=0; cc<2; cc++) { if (mid_mask_1 + mid_mask_2 > 0.0) { @@ -520,7 +524,7 @@ TEST_CASE("io_remap_test","io_remap_test") test_int = mask_val; } REQUIRE(approx(Vm_v_vh(ii,cc,jj), test_mid)); - REQUIRE(approx(Vi_v_vh(ii,cc,jj), test_int)); + REQUIRE(approx(Vi_v_vh(ii,cc,jj), test_int)); } } // For the pressured sliced variable we expect it to match the solution from horizontal mapping only so we use the same syntax. @@ -531,7 +535,7 @@ TEST_CASE("io_remap_test","io_remap_test") found = true; Ys_exp += calculate_output(p_ref,col1,0)*wgt; Ys_wgt += wgt; - } + } if (p_ref<=pi_v(col2,nlevs_src) && p_ref>=pi_v(col2,0)) { found = true; Ys_exp += calculate_output(p_ref,col2,0)*(1.0-wgt); @@ -547,7 +551,7 @@ TEST_CASE("io_remap_test","io_remap_test") print (" -> vertical + horizontal remap ... done\n",io_comm); } // ------------------------------------------------------------------------------------------------------ - // All Done + // All Done print (" -> Test Remapped Output ... done\n",io_comm); scorpio::finalize_subsystem(); @@ -677,7 +681,6 @@ ekat::ParameterList set_output_params(const std::string& name, const std::string params.set("Averaging Type","Instant"); params.set("Max Snapshots Per File",1); params.set("Floating Point Precision","real"); - params.set("MPI Ranks in Filename",true); auto& oc = params.sublist("output_control"); oc.set("Frequency",1); oc.set("frequency_units","nsteps"); @@ -695,12 +698,12 @@ ekat::ParameterList set_output_params(const std::string& name, const std::string if (vert_remap) { params.set("vertical_remap_file",remap_filename); // TODO, make this work for general np=? - } + } if (horiz_remap) { params.set("horiz_remap_file",remap_filename); // TODO, make this work for general np=? - } - - return params; + } + + return params; } /*==========================================================================================================*/ ekat::ParameterList set_input_params(const std::string& name, ekat::Comm& comm, const std::string& tstamp, const int p_ref) @@ -716,7 +719,7 @@ ekat::ParameterList set_input_params(const std::string& name, ekat::Comm& comm, in_params.set("Field Names", fields_in); in_params.set("Floating Point Precision","real"); - return in_params; + return in_params; } /*==========================================================================================================*/ diff --git a/components/eamxx/src/share/io/tests/io_se_grid.cpp b/components/eamxx/src/share/io/tests/io_se_grid.cpp index 36acf2a833cd..0e6d28b53d62 100644 --- a/components/eamxx/src/share/io/tests/io_se_grid.cpp +++ b/components/eamxx/src/share/io/tests/io_se_grid.cpp @@ -64,13 +64,13 @@ TEST_CASE("se_grid_io") params.set("Max Snapshots Per File",1); params.set("Field Names",{"field_1","field_2","field_3","field_packed"}); params.set("Floating Point Precision","real"); - params.set("MPI Ranks in Filename",true); auto& ctl_pl = params.sublist("output_control"); ctl_pl.set("Frequency",1); ctl_pl.set("frequency_units","nsteps"); OutputManager om; - om.setup(io_comm,params,fm0,gm,t0,t0,false); + om.initialize(io_comm,params,t0,false); + om.setup(fm0,gm); om.init_timestep(t0,dt); om.run(t0+dt); om.finalize(); @@ -95,7 +95,7 @@ TEST_CASE("se_grid_io") } ins_input.finalize(); - // All Done + // All Done scorpio::finalize_subsystem(); } @@ -152,8 +152,9 @@ get_test_fm(const std::shared_ptr& grid, // field_2 is not partitioned, so let's sync it across ranks auto f2 = fm->get_field("field_2"); - auto v2 = f2.get_view(); + auto v2 = f2.get_view(); comm.all_reduce(v2.data(),nlevs,MPI_MAX); + f2.sync_to_dev(); return fm; } diff --git a/components/eamxx/src/share/io/tests/output_restart.cpp b/components/eamxx/src/share/io/tests/output_restart.cpp index 8eea8c97f93c..30dfa1b4270b 100644 --- a/components/eamxx/src/share/io/tests/output_restart.cpp +++ b/components/eamxx/src/share/io/tests/output_restart.cpp @@ -78,7 +78,6 @@ TEST_CASE("output_restart","io") output_params.set("Floating Point Precision","real"); output_params.set>("Field Names",{"field_1", "field_2", "field_3", "field_4","field_5"}); output_params.set("fill_value",FillValue); - output_params.set("MPI Ranks in Filename","true"); output_params.set("flush_frequency",1); output_params.sublist("output_control").set("frequency_units","nsteps"); output_params.sublist("output_control").set("Frequency",10); @@ -93,7 +92,8 @@ TEST_CASE("output_restart","io") const int nsteps) { OutputManager output_manager; - output_manager.setup(comm,output_params,fm,gm,run_t0,case_t0,false); + output_manager.initialize(comm, output_params, run_t0, case_t0, false); + output_manager.setup(fm,gm); // We advance the fields, by adding dt to each entry of the fields at each time step // The output restart data is written every 5 time steps, while the output freq is 10. @@ -132,7 +132,7 @@ TEST_CASE("output_restart","io") output_params.set("filename_prefix","monolithic"); output_params.sublist("Checkpoint Control").set("frequency_units","never"); run(fm_mono,t0,t0,20); - + // 2. Run for 15 days on fm0, write restart every 5 steps auto fm_rest = clone_fm(fm0); output_params.set("filename_prefix","restarted"); @@ -151,7 +151,7 @@ TEST_CASE("output_restart","io") } // Finalize everything scorpio::finalize_subsystem(); -} +} /*=============================================================================================*/ std::shared_ptr diff --git a/components/eamxx/src/share/property_checks/field_nan_check.cpp b/components/eamxx/src/share/property_checks/field_nan_check.cpp index eef52adfbf78..5b709ff916de 100644 --- a/components/eamxx/src/share/property_checks/field_nan_check.cpp +++ b/components/eamxx/src/share/property_checks/field_nan_check.cpp @@ -186,6 +186,7 @@ PropertyCheck::ResultAndMsg FieldNaNCheck::check() const { "Internal error in FieldNaNCheck: unsupported field data type.\n" "You should not have reached this line. Please, contact developers.\n"); } + return ResultAndMsg{}; } } // namespace scream diff --git a/components/eamxx/src/share/property_checks/field_within_interval_check.cpp b/components/eamxx/src/share/property_checks/field_within_interval_check.cpp index bd436bb192a4..eee5f96855d5 100644 --- a/components/eamxx/src/share/property_checks/field_within_interval_check.cpp +++ b/components/eamxx/src/share/property_checks/field_within_interval_check.cpp @@ -332,6 +332,7 @@ PropertyCheck::ResultAndMsg FieldWithinIntervalCheck::check() const { "Internal error in FieldWithinIntervalCheck: unsupported field data type.\n" "You should not have reached this line. Please, contact developers.\n"); } + return ResultAndMsg{}; } template diff --git a/components/eamxx/src/share/scream_types.hpp b/components/eamxx/src/share/scream_types.hpp index 5d0dacf86211..cbe26aae75f3 100644 --- a/components/eamxx/src/share/scream_types.hpp +++ b/components/eamxx/src/share/scream_types.hpp @@ -34,12 +34,13 @@ enum class RepoState { // The type of this run enum class RunType { Initial, - Restarted + Restart }; // We cannot expect BFB results between f90 and cxx if optimizations are on. // Same goes for cuda-memcheck because it makes the bfb math layer prohibitively -// expensive and so must be turned off. +// expensive and so must be turned off. SCREAM_SHORT_TESTS is a proxy +// for mem checking. #if defined (NDEBUG) || defined (SCREAM_SHORT_TESTS) static constexpr bool SCREAM_BFB_TESTING = false; #else diff --git a/components/eamxx/src/share/tests/CMakeLists.txt b/components/eamxx/src/share/tests/CMakeLists.txt index 3b4b60cb283b..c64659575cba 100644 --- a/components/eamxx/src/share/tests/CMakeLists.txt +++ b/components/eamxx/src/share/tests/CMakeLists.txt @@ -51,10 +51,16 @@ if (NOT SCREAM_ONLY_GENERATE_BASELINES) LIBS scream_io MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS}) - # Test vertical remap - CreateUnitTest(time_interpolation "eamxx_time_interpolation_tests.cpp" + # Generate data for data interpolation test + CreateUnitTest(data_interpolation_setup "data_interpolation_setup.cpp" LIBS scream_io - MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS}) + FIXTURES_SETUP data_interpolation_setup) + + # Test data interpolation + CreateUnitTest(data_interpolation "data_interpolation_tests.cpp" + LIBS scream_io + MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} + FIXTURES_REQUIRED data_interpolation_setup) # Test common physics functions CreateUnitTest(common_physics "common_physics_functions_tests.cpp") diff --git a/components/eamxx/src/share/tests/data_interpolation_setup.cpp b/components/eamxx/src/share/tests/data_interpolation_setup.cpp new file mode 100644 index 000000000000..3ce20351312e --- /dev/null +++ b/components/eamxx/src/share/tests/data_interpolation_setup.cpp @@ -0,0 +1,170 @@ +#include + +#include "data_interpolation_tests.hpp" + +#include "share/io/scream_io_utils.hpp" +#include "share/io/scream_scorpio_interface.hpp" +#include "share/grid/point_grid.hpp" + +namespace scream { + +TEST_CASE ("data_interpolation_setup") +{ + // NOTE: ensure these match what's used in data_interpolation_tests.cpp + constexpr int ngcols = data_ngcols; + constexpr int nlevs = data_nlevs; + + auto t_ref = get_t_ref(); + + // Init test session + ekat::Comm comm(MPI_COMM_WORLD); + scorpio::init_subsystem(comm); + + // We use raw scorpio calls without decomp, so ensure we're in serial case + EKAT_REQUIRE_MSG (comm.size()==1, + "Error! You should run the data_interpolation_setup test with ONE rank.\n"); + + // Create grid + std::shared_ptr grid = create_point_grid("pg",ngcols,nlevs,comm); + + // Create and setup two files, so we can test both YearlyPeriodic and LinearHistory + std::vector files = { + "data_interpolation_0", + "data_interpolation_1" + }; + + for (auto int_same_as_mid : {true, false}) { + auto suffix = int_same_as_mid ? "_no_ilev.nc" : ".nc"; + for (const std::string& fname : files) { + scorpio::register_file(fname+suffix,scorpio::Write); + + scorpio::define_dim (fname+suffix,"ncol",ngcols); + scorpio::define_dim (fname+suffix,"lev",nlevs); + scorpio::define_dim (fname+suffix,"dim2",ncmps); + scorpio::define_time(fname+suffix,"days since " + t_ref.to_string()); + if (not int_same_as_mid) { + scorpio::define_dim (fname+suffix,"ilev",nlevs+1); + } + + std::string ilev = int_same_as_mid ? "lev" : "ilev"; + + scorpio::define_var(fname+suffix,"s2d", {"ncol"}, "real", true); + scorpio::define_var(fname+suffix,"s2d", {"ncol"}, "real", true); + scorpio::define_var(fname+suffix,"v2d", {"ncol","dim2"}, "real", true); + scorpio::define_var(fname+suffix,"s3d_m",{"ncol","lev"}, "real", true); + scorpio::define_var(fname+suffix,"v3d_m",{"ncol","dim2","lev"},"real", true); + scorpio::define_var(fname+suffix,"s3d_i",{"ncol",ilev}, "real", true); + scorpio::define_var(fname+suffix,"v3d_i",{"ncol","dim2",ilev}, "real", true); + + // We keep p1d and p3d NOT time-dep + scorpio::define_var(fname+suffix,"p1d", {"lev"},"real", false); + scorpio::define_var(fname+suffix,"p3d", {"ncol","lev"},"real", false); + + scorpio::enddef(fname+suffix); + } + + // Fields and some helper fields (for later) + // NOTE: if we save a pressure field, there is not distinction + // between interfaces and midpoints in the file + // NOTE: do not pad, so that we can grab pointers and pass them to scorpio + auto base_fields = create_fields(grid,true, int_same_as_mid,false); + auto fields = create_fields(grid,false,int_same_as_mid,false); + auto ones = create_fields(grid,false,int_same_as_mid,false); + for (const auto& f : ones) { + f.deep_copy(1); + } + // Loop over time, and add 30 to the value for the first 6 months, + // and subtract 30 for the last 6 months. This guarantees that the data + // is indeed periodic. We'll write at the 15th of each month + // Generate three files: + // - one to be used for yearly-periodic interp + // - two to be used for linear-hystory interp + util::TimeStamp time = get_first_slice_time (); + + // We keep p1d and p3d NOT time-dep, so we write outside the loop + auto p1d = base_fields.back(); + auto p3d = base_fields[2].alias("p3d"); + p1d.sync_to_host(); + p3d.sync_to_host(); + for (const std::string& fname : files) { + scorpio::write_var(fname+suffix,p1d.name(),p1d.get_internal_view_data()); + scorpio::write_var(fname+suffix,p3d.name(),p3d.get_internal_view_data()); + } + + int nfields = fields.size() - 1; // Don't handle p1d, since it's done above + for (int mm=0; mm<12; ++mm) { + std::string file_name = "data_interpolation_" + std::to_string(mm/6) + suffix; + + // We start the files with July + int mm_index = mm+6; + scorpio::update_time(file_name,time.days_from(t_ref)); + for (int i=0; i()); + } + time += 86400*time.days_in_curr_month(); + } + + for (const std::string& fname : files) { + write_timestamp(fname+suffix,"reference_time_stamp",t_ref); + scorpio::release_file(fname+suffix); + } + } + + // Now write a map file for horiz remap, that splits each dof interval in two + const int ngdofs_src = data_ngcols; + const int ngdofs_tgt = fine_ngcols; + + // Existing dofs are "copied", added dofs are averaged from neighbors + const int nnz = ngdofs_src + 2*(ngdofs_src-1); + + std::string filename = map_file_name; + scorpio::register_file(filename, scorpio::FileMode::Write); + + scorpio::define_dim(filename, "n_a", ngdofs_src); + scorpio::define_dim(filename, "n_b", ngdofs_tgt); + scorpio::define_dim(filename, "n_s", nnz); + + scorpio::define_var(filename, "col", {"n_s"}, "int"); + scorpio::define_var(filename, "row", {"n_s"}, "int"); + scorpio::define_var(filename, "S", {"n_s"}, "double"); + + scorpio::enddef(filename); + + std::vector col(nnz), row(nnz); + std::vector S(nnz); + for (int i=0,nnz=0; i + +#include "data_interpolation_tests.hpp" + +#include "share/io/scream_scorpio_interface.hpp" +#include "share/util/eamxx_data_interpolation.hpp" +#include "share/grid/point_grid.hpp" +#include "share/field/field_utils.hpp" +#include "share/scream_config.hpp" + +namespace scream { + +// Give ourselves some room for roundoff errors, since our manual +// evaluation may be different (in finite prec) from the one in the class. +constexpr auto tol = std::numeric_limits::epsilon()*10; + +constexpr auto P1D = DataInterpolation::Static1D; +constexpr auto P3D = DataInterpolation::Dynamic3D; + +using strvec_t = std::vector; +using namespace ShortFieldTagsNames; + +util::TimeStamp reset_year (const util::TimeStamp& t_in, int yy) +{ + auto date = t_in.get_date(); + auto time = t_in.get_time(); + date[0] = yy; + return util::TimeStamp(date,time); +} + +std::shared_ptr +create_interp (const std::shared_ptr& grid, + const std::vector& fields) +{ + return std::make_shared(grid,fields); +} + +void root_print (const ekat::Comm& comm, + const std::string& msg) +{ + if (comm.am_i_root()) { + printf("%s",msg.c_str()); + } +} + +// Run the data interpolation to the input grid, and check against expected values +void run_tests (const std::shared_ptr& grid, + const strvec_t& input_files, util::TimeStamp t_beg, + const util::TimeLine timeline, + const DataInterpolation::VRemapType vr_type = DataInterpolation::None) +{ + auto t_end = t_beg + t_beg.days_in_curr_month()*spd; + auto t0 = t_beg + (t_end-t_beg)/2; + + auto ncols = grid->get_num_local_dofs(); + auto nlevs = grid->get_num_vertical_levels(); + + auto vcoarse_grid = grid->clone("vcoarse",true); + vcoarse_grid->reset_num_vertical_lev(data_nlevs); + + // These are the fields we will compute + auto fields = create_fields(grid,false,false); + fields.pop_back(); // We don't interpolate p1d... + + std::string map_file = grid->get_num_global_dofs()==data_ngcols ? "" : map_file_name; + + // These are used to check the answer + auto base = create_fields(grid,true); + auto ones = create_fields(grid,false); + auto diff = create_fields(grid,false); + auto expected = create_fields(grid,false); + for (auto& f : ones) { + f.deep_copy(1); + } + + std::string data_pname = vr_type==P1D ? "p1d" : "p3d"; // if vr_type==None, it's not used anyways + auto model_pmid = base[2].clone("pmid"); + auto model_pint = base[4].clone("pint"); + if (vr_type==P1D) { + // It's complicated to test the static profile, since we'd have to really do + // a manual interpolation. But setting all model pressure equal to the 1st col + // of the pressure makes things doable + auto comm = grid->get_comm(); + for (const Field& p : {model_pmid, model_pint}) { + auto col_0 = p.subfield(0,0); + auto len = col_0.get_header().get_identifier().get_layout().size(); + comm.broadcast(col_0.get_internal_view_data(),len,0); + col_0.sync_to_dev(); + + for (int icol=0; icol expected_vcoarse, base_vcoarse, ones_vcoarse; + if (vr_type==P1D or vr_type==P3D) { + // If we do remap, there is some P0 extrapolation, + // for which we need to know the data at the top/bot + // NOTE: the data has NO ilev coord in this case + expected_vcoarse = create_fields(vcoarse_grid,false,true); + base_vcoarse = create_fields(vcoarse_grid,true,true); + ones_vcoarse = create_fields(vcoarse_grid,false,true); + for (auto& f : ones_vcoarse) { + f.deep_copy(1); + } + } + + int nfields = fields.size(); + auto interp = create_interp(grid,fields); + interp->setup_time_database(input_files,util::TimeLine::YearlyPeriodic); + interp->setup_remappers (map_file,vr_type,data_pname,model_pmid,model_pint); + interp->init_data_interval(t0); + + // We jump ahead by 2 months, but the shift interval logic cannot keep up with + // a dt that long, so we should get an error due to the interpolation param being + // outside the [0,1] interval. + REQUIRE_THROWS (interp->run(t0+60*spd)); + + // Loop for two year at a 20 day increment + int dt = 20*spd; + for (auto time = t0+dt; time.days_from(t0)<365; time+=dt) { + if (t_end1) { + std::cout << "TEST ERROR:\n" + << " t beg: " << t_beg.to_string() << "\n" + << " t end: " << t_end.to_string() << "\n" + << " time : " << time.to_string() << "\n" + << " t-beg: " << time_from_beg.length << "\n" + << " days in mm_beg: " << t_beg.days_in_curr_month() << "\n" + << " alpha: " << alpha << "\n" + << " delta_beg: " << delta_data[mm_beg] << "\n" + << " delta_end: " << delta_data[mm_end] << "\n" + << " delta: " << delta << "\n"; + } + // Compute expected difference from base value + interp->run(time); + for (int i=0; i(); + auto e_vcoarse = expected_vcoarse[i].get_view(); + + for (int icol=0; icol(); + auto e_vcoarse = expected_vcoarse[i].get_view(); + + for (int icol=0; icol(expected[i])); + REQUIRE (frobenius_norm(diff[i])setup_time_database(files,util::TimeLine::Linear)); // Input file not readable + + interp->setup_time_database({"./data_interpolation_0.nc"},util::TimeLine::Linear); + util::TimeStamp t0 ({2000,1,1},{0,0,0}); + REQUIRE_THROWS (interp->init_data_interval(t0)); // linear timeline, but t0init_data_interval(t1)); // linear timeline, but t0>last_slice + + scorpio::finalize_subsystem(); +} + +TEST_CASE ("interpolation") +{ + ekat::Comm comm(MPI_COMM_WORLD); + + // Regardless of how EAMxx is configured, ignore leap years for this test + set_use_leap_year(false); + + scorpio::init_subsystem(comm); + + auto data_grid = create_point_grid("pg",data_ngcols,data_nlevs,comm); + auto hfine_grid = create_point_grid("pg_h",fine_ngcols,data_nlevs,comm); + auto vfine_grid = create_point_grid("pg_v",data_ngcols,fine_nlevs,comm); + auto hvfine_grid = create_point_grid("pg_hv",fine_ngcols,fine_nlevs,comm); + + SECTION ("periodic") { + // We assume we start with t0 sometime between the first and second input slice. + auto t_beg = reset_year(get_last_slice_time(),2019); + auto timeline = util::TimeLine::YearlyPeriodic; + strvec_t files = {"data_interpolation_0.nc","data_interpolation_1.nc"}; + strvec_t files_no_ilev = {"data_interpolation_0_no_ilev.nc","data_interpolation_1_no_ilev.nc"}; + + SECTION ("no-horiz") { + SECTION ("no-vert") { + root_print(comm," timeline=PERIODIC, horiz_remap=NO, vert_remap=NO ..........\n"); + run_tests (data_grid,files,t_beg,timeline); + root_print(comm," timeline=PERIODIC, horiz_remap=NO, vert_remap=NO .......... PASS\n"); + } + SECTION ("p1d-vert") { + root_print(comm," timeline=PERIODIC, horiz_remap=NO, vert_remap=p1d .........\n"); + run_tests (vfine_grid,files_no_ilev,t_beg,timeline,P1D); + root_print(comm," timeline=PERIODIC, horiz_remap=NO, vert_remap=p1d ......... PASS\n"); + } + SECTION ("p3d-vert") { + root_print(comm," timeline=PERIODIC, horiz_remap=NO, vert_remap=p3d .........\n"); + run_tests (vfine_grid,files_no_ilev,t_beg,timeline,P3D); + root_print(comm," timeline=PERIODIC, horiz_remap=NO, vert_remap=p3d ......... PASS\n"); + } + } + + SECTION ("yes-horiz") { + SECTION ("no-vert") { + root_print(comm," timeline=PERIODIC, horiz_remap=YES, vert_remap=NO ..........\n"); + run_tests (hfine_grid,files,t_beg,timeline); + root_print(comm," timeline=PERIODIC, horiz_remap=YES, vert_remap=NO .......... PASS\n"); + } + SECTION ("p1d-vert") { + root_print(comm," timeline=PERIODIC, horiz_remap=YES, vert_remap=p1d .........\n"); + run_tests (hvfine_grid,files_no_ilev,t_beg,timeline,P1D); + root_print(comm," timeline=PERIODIC, horiz_remap=YES, vert_remap=p1d ......... PASS\n"); + } + SECTION ("p3d-vert") { + root_print(comm," timeline=PERIODIC, horiz_remap=YES, vert_remap=p3d .........\n"); + run_tests (hvfine_grid,files_no_ilev,t_beg,timeline,P3D); + root_print(comm," timeline=PERIODIC, horiz_remap=YES, vert_remap=p3d ......... PASS\n"); + } + } + } + + SECTION ("linear") { + // We assume we start with t0 sometime between the first and second input slice. + auto t_beg = get_first_slice_time(); + auto timeline = util::TimeLine::Linear; + strvec_t files = {"data_interpolation_0.nc","data_interpolation_1.nc"}; + strvec_t files_no_ilev = {"data_interpolation_0_no_ilev.nc","data_interpolation_1_no_ilev.nc"}; + + SECTION ("no-horiz") { + SECTION ("no-vert") { + root_print(comm," timeline=LINEAR, horiz_remap=NO, vert_remap=NO ..........\n"); + run_tests (data_grid,files,t_beg,timeline); + root_print(comm," timeline=LINEAR, horiz_remap=NO, vert_remap=NO .......... PASS\n"); + } + SECTION ("p1d-vert") { + root_print(comm," timeline=LINEAR, horiz_remap=NO, vert_remap=p1d .........\n"); + run_tests (vfine_grid,files_no_ilev,t_beg,timeline,P1D); + root_print(comm," timeline=LINEAR, horiz_remap=NO, vert_remap=p1d ......... PASS\n"); + } + SECTION ("p3d-vert") { + root_print(comm," timeline=LINEAR, horiz_remap=NO, vert_remap=p3d .........\n"); + run_tests (vfine_grid,files_no_ilev,t_beg,timeline,P3D); + root_print(comm," timeline=LINEAR, horiz_remap=NO, vert_remap=p3d ......... PASS\n"); + } + } + + SECTION ("yes-horiz") { + SECTION ("no-vert") { + root_print(comm," timeline=LINEAR, horiz_remap=YES, vert_remap=NO ..........\n"); + run_tests (hfine_grid,files,t_beg,timeline); + root_print(comm," timeline=LINEAR, horiz_remap=YES, vert_remap=NO .......... PASS\n"); + } + SECTION ("p1d-vert") { + root_print(comm," timeline=LINEAR, horiz_remap=YES, vert_remap=p1d .........\n"); + run_tests (hvfine_grid,files_no_ilev,t_beg,timeline,P1D); + root_print(comm," timeline=LINEAR, horiz_remap=YES, vert_remap=p1d ......... PASS\n"); + } + SECTION ("p3d-vert") { + root_print(comm," timeline=LINEAR, horiz_remap=YES, vert_remap=p3d .........\n"); + run_tests (hvfine_grid,files_no_ilev,t_beg,timeline,P3D); + root_print(comm," timeline=LINEAR, horiz_remap=YES, vert_remap=p3d ......... PASS\n"); + } + } + } + + scorpio::finalize_subsystem(); +} + +} // anonymous namespace diff --git a/components/eamxx/src/share/tests/data_interpolation_tests.hpp b/components/eamxx/src/share/tests/data_interpolation_tests.hpp new file mode 100644 index 000000000000..fc2c784c8f17 --- /dev/null +++ b/components/eamxx/src/share/tests/data_interpolation_tests.hpp @@ -0,0 +1,159 @@ +#ifndef EAMXX_DATA_INTERPOLATION_TESTS_HPP +#define EAMXX_DATA_INTERPOLATION_TESTS_HPP + +#include "share/grid/abstract_grid.hpp" +#include "share/field/field.hpp" +#include "share/util/scream_universal_constants.hpp" + +#include + +namespace scream +{ + +constexpr int ncmps = 2; +constexpr auto spd = constants::seconds_per_day; +constexpr int data_ngcols = 12; +constexpr int fine_ngcols = 2*data_ngcols-1; // stick one dof between each data dofs +constexpr int data_nlevs = 32; +constexpr int fine_nlevs = 64; +const std::string map_file_name = "map_file_for_data_interp.nc"; + +// At each month in the input data, we are adding a delta to the "base" value of the fields. +constexpr double delta_data[12] = {0, 30, 60, 90, 120, 150, 180, 150, 120, 90, 60, 30}; + +inline util::TimeStamp get_t_ref () { + return util::TimeStamp ({2010,1,1},{0,0,0}); +} + +// Slices are at midnight between 15th and 16th of each month +// First slice is Jul 15th +inline util::TimeStamp get_first_slice_time () { + // 15 days after the reference time + auto t = get_t_ref() + spd*15; // Mid Jan + for (int mm=0; mm<6; ++mm) { + t += spd*t.days_in_curr_month(); + } + return t; +} + +inline util::TimeStamp get_last_slice_time () { + // 11 months after the 1st slice + auto t = get_first_slice_time(); + for (int mm=0; mm<11; ++mm) { + t += spd*t.days_in_curr_month(); + } + return t; +} + +std::vector +create_fields (const std::shared_ptr& grid, + const bool init_values, + const bool int_same_as_mid = false, + const bool pad_for_packing = true) +{ + constexpr auto m = ekat::units::m; + const auto& gn = grid->name(); + + int ncols = grid->get_num_local_dofs(); + int nlevs = grid->get_num_vertical_levels(); + + // Create fields + + auto layout_s2d = grid->get_2d_scalar_layout(); + auto layout_v2d = grid->get_2d_vector_layout(ncmps); + auto layout_s3d_m = grid->get_3d_scalar_layout(true); + auto layout_v3d_m = grid->get_3d_vector_layout(true,ncmps); + auto layout_s3d_i = grid->get_3d_scalar_layout(int_same_as_mid); + auto layout_v3d_i = grid->get_3d_vector_layout(int_same_as_mid,ncmps); + + Field s2d (FieldIdentifier("s2d", layout_s2d, m, gn)); + Field v2d (FieldIdentifier("v2d", layout_v2d, m, gn)); + Field s3d_m(FieldIdentifier("s3d_m", layout_s3d_m, m, gn)); + Field v3d_m(FieldIdentifier("v3d_m", layout_v3d_m, m, gn)); + Field s3d_i(FieldIdentifier("s3d_i", layout_s3d_i, m, gn)); + Field v3d_i(FieldIdentifier("v3d_i", layout_v3d_i, m, gn)); + + if (pad_for_packing) { + s3d_m.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); + v3d_m.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); + s3d_i.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); + v3d_i.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); + } + + s2d.allocate_view(); + v2d.allocate_view(); + s3d_m.allocate_view(); + v3d_m.allocate_view(); + s3d_i.allocate_view(); + v3d_i.allocate_view(); + + if (init_values) { + // We set horiz/vert values based on the position of the dof, assuming that + // - we have a 1d horiz grid + // - leftmost h dof gets a value of 0, rightmost a value of 1.0 + // - bottom interface gets a value of 0, top interface get a value of 1.0 + // - midpoints are the avg of interfaces + // - we do h_value*v_value + icmp + int ngcols = grid->get_num_global_dofs(); + int nh_intervals = ngcols - 1; + double h_value, v_value; + double h_max = 1.0; + double v_max = 1.0; + double dh = h_max / nh_intervals; + double dv = v_max / nlevs; + auto gids = grid->get_dofs_gids().get_view(); + for (int icol=0; icol()(icol,icmp,ilev) = h_value*(v_value + dv/2) + icmp; + if (int_same_as_mid) { + v3d_i.get_view()(icol,icmp,ilev) = h_value*(v_value + dv/2) + icmp; + } else { + v3d_i.get_view()(icol,icmp,ilev) = h_value*(v_value) + icmp; + } + } + s3d_m.get_view()(icol,ilev) = h_value*(v_value + dv/2); + if (int_same_as_mid) { + s3d_i.get_view()(icol,ilev) = h_value*(v_value + dv/2); + } else { + s3d_i.get_view()(icol,ilev) = h_value*(v_value); + } + } + // Last interface (if mid!=int), where v_value=1 + if (not int_same_as_mid) { + s3d_i.get_view()(icol,nlevs) = h_value; + for (int icmp=0; icmp()(icol,icmp,nlevs) = h_value + icmp; + } + } + + // 2D quantities + for (int icmp=0; icmp()(icol,icmp) = h_value + icmp; + } + s2d.get_view()(icol) = h_value; + } + + s2d.sync_to_dev(); + v2d.sync_to_dev(); + s3d_m.sync_to_dev(); + v3d_m.sync_to_dev(); + s3d_i.sync_to_dev(); + v3d_i.sync_to_dev(); + } + + auto p1d = s3d_m.subfield(0,0).clone("p1d"); + auto comm = grid->get_comm(); + comm.broadcast(p1d.get_internal_view_data(),nlevs,0); + p1d.sync_to_dev(); + + return {s2d, v2d, s3d_m, v3d_m, s3d_i, v3d_i, p1d}; +} + +} // namespace scream + +#endif // EAMXX_DATA_INTERPOLATION_TESTS_HPP diff --git a/components/eamxx/src/share/tests/eamxx_time_interpolation_tests.cpp b/components/eamxx/src/share/tests/eamxx_time_interpolation_tests.cpp index c8551d081266..238887e909aa 100644 --- a/components/eamxx/src/share/tests/eamxx_time_interpolation_tests.cpp +++ b/components/eamxx/src/share/tests/eamxx_time_interpolation_tests.cpp @@ -75,7 +75,7 @@ TEST_CASE ("eamxx_time_interpolation_simple") { // Setup basic test params ekat::Comm comm(MPI_COMM_WORLD); auto seed = get_random_test_seed(&comm); - std::mt19937_64 engine(seed); + std::mt19937_64 engine(seed); const auto t0 = init_timestamp(); const int nlevs = SCREAM_PACK_SIZE*2+1; @@ -141,7 +141,7 @@ TEST_CASE ("eamxx_time_interpolation_data_from_file") { ekat::Comm comm(MPI_COMM_WORLD); scorpio::init_subsystem(comm); auto seed = get_random_test_seed(&comm); - std::mt19937_64 engine(seed); + std::mt19937_64 engine(seed); const auto t0 = init_timestamp(); const int nlevs = SCREAM_PACK_SIZE*2+1; @@ -332,7 +332,7 @@ std::shared_ptr get_fm (const std::shared_ptr& // - Uniform_int_distribution returns an int, and the randomize // util checks that return type matches the Field data type. // So wrap the int pdf in a lambda, that does the cast. - std::mt19937_64 engine(seed); + std::mt19937_64 engine(seed); const int nlcols = grid->get_num_local_dofs(); const int nlevs = grid->get_num_vertical_levels(); @@ -367,7 +367,7 @@ std::shared_ptr get_fm (const std::shared_ptr& * the capability of TimeInterpolation to handle data read from multiple files. */ std::vector create_test_data_files( - const ekat::Comm& comm, + const ekat::Comm& comm, const std::shared_ptr& gm, const util::TimeStamp& t0, const int seed) @@ -385,7 +385,6 @@ std::vector create_test_data_files( } // Create the output parameters ekat::ParameterList om_pl; - om_pl.set("MPI Ranks in Filename",true); om_pl.set("filename_prefix",std::string("source_data_for_time_interpolation")); om_pl.set("Field Names",fnames); om_pl.set("Averaging Type", std::string("INSTANT")); @@ -393,12 +392,12 @@ std::vector create_test_data_files( auto& ctrl_pl = om_pl.sublist("output_control"); ctrl_pl.set("frequency_units",std::string("nsteps")); ctrl_pl.set("Frequency",snap_freq); - ctrl_pl.set("MPI Ranks in Filename",true); ctrl_pl.set("save_grid_data",false); // Create an output manager, note we use a subclass defined in this test so we can extract // the list of files created by the output manager. OutputManager4Test om; - om.setup(comm,om_pl,fm,gm,t0,false); + om.initialize(comm,om_pl,t0,false); + om.setup(fm,gm); // Time loop to create and write data auto tw = t0; diff --git a/components/eamxx/src/share/tests/field_tests.cpp b/components/eamxx/src/share/tests/field_tests.cpp index 0dfe2522de39..38a3b04c2c66 100644 --- a/components/eamxx/src/share/tests/field_tests.cpp +++ b/components/eamxx/src/share/tests/field_tests.cpp @@ -261,6 +261,17 @@ TEST_CASE("field", "") { } SECTION ("deep_copy") { + // rank-0 + std::vector t0 = {}; + std::vector d0 = {}; + FieldIdentifier fid0("scalar_0d",{t0,d0},m/s,"some_grid"); + Field f0(fid0); + f0.allocate_view(); + f0.deep_copy(1.5); + f0.sync_to_host(); + REQUIRE (reinterpret_cast(f0.get_internal_view_data())[0]==1.5); + + // rank-3 std::vector t1 = {COL,CMP,LEV}; std::vector d1 = {3,2,24}; diff --git a/components/eamxx/src/share/tests/field_utils.cpp b/components/eamxx/src/share/tests/field_utils.cpp index ffc27a3b0073..4ba0a8294e4a 100644 --- a/components/eamxx/src/share/tests/field_utils.cpp +++ b/components/eamxx/src/share/tests/field_utils.cpp @@ -37,6 +37,31 @@ TEST_CASE("utils") { f1.get_header().get_alloc_properties().request_allocation(P8::n); f1.allocate_view(); + SECTION("compare-rank-0") { + // create two fields with rank-0 and get their views + std::vector tags_0 = {}; + std::vector dims_0 = {}; + FieldIdentifier fid_01("field_01", {tags_0, dims_0}, m / s, "some_grid"); + FieldIdentifier fid_02("field_02", {tags_0, dims_0}, m / s, "some_grid"); + Field f01(fid_01); + Field f02(fid_02); + f01.allocate_view(); + f02.allocate_view(); + auto f01v = f01.get_view(); + auto f02v = f02.get_view(); + // fill the views with the same values + Real val = 54321; + Kokkos::deep_copy(f01v, val); + Kokkos::deep_copy(f02v, val); + // check that the views are equal + REQUIRE(views_are_equal(f01, f02)); + + // fill the views with different values + Kokkos::deep_copy(f02v, 1 / val); + // check that the views are not equal + REQUIRE(not views_are_equal(f01, f02)); + } + SECTION ("compare") { Field f2(fid); @@ -101,6 +126,288 @@ TEST_CASE("utils") { REQUIRE(field_sum(f1,&comm)==gsum); } + SECTION("horiz_contraction") { + using RPDF = std::uniform_real_distribution; + auto engine = setup_random_test(); + RPDF pdf(0, 1); + + int dim0 = 3; + int dim1 = 9; + int dim2 = 2; + + // Set a weight field + FieldIdentifier f00("f", {{COL}, {dim0}}, m / s, "g"); + Field field00(f00); + field00.allocate_view(); + field00.sync_to_host(); + auto v00 = field00.get_strided_view(); + for(int i = 0; i < dim0; ++i) { + v00(i) = (i + 1) / sp(6); + } + field00.sync_to_dev(); + + // Create (random) sample fields + FieldIdentifier fsc("f", {{}, {}}, m / s, "g"); // scalar + FieldIdentifier f10("f", {{COL, CMP}, {dim0, dim1}}, m / s, "g"); + FieldIdentifier f11("f", {{COL, LEV}, {dim0, dim2}}, m / s, "g"); + FieldIdentifier f20("f", {{COL, CMP, LEV}, {dim0, dim1, dim2}}, m / s, "g"); + Field fieldsc(fsc); + Field field10(f10); + Field field11(f11); + Field field20(f20); + fieldsc.allocate_view(); + field10.allocate_view(); + field11.allocate_view(); + field20.allocate_view(); + randomize(fieldsc, engine, pdf); + randomize(field10, engine, pdf); + randomize(field11, engine, pdf); + randomize(field20, engine, pdf); + + FieldIdentifier F_x("fx", {{COL}, {dim0}}, m / s, "g"); + FieldIdentifier F_y("fy", {{LEV}, {dim2}}, m / s, "g"); + FieldIdentifier F_z("fz", {{CMP}, {dim1}}, m / s, "g"); + FieldIdentifier F_w("fyz", {{CMP, LEV}, {dim1, dim2}}, m / s, "g"); + + Field field_x(F_x); + Field field_y(F_y); + Field field_z(F_z); + Field field_w(F_w); + + // Test invalid inputs + REQUIRE_THROWS(horiz_contraction(fieldsc, field_x, + field00)); // x not allocated yet + + field_x.allocate_view(); + field_y.allocate_view(); + field_z.allocate_view(); + field_w.allocate_view(); + + REQUIRE_THROWS(horiz_contraction(fieldsc, field_y, + field_x)); // unmatching layout + REQUIRE_THROWS(horiz_contraction(field_z, field11, + field11)); // wrong weight layout + + Field result; + + // Ensure a scalar case works + result = fieldsc.clone(); + horiz_contraction(result, field00, field00); + result.sync_to_host(); + auto v = result.get_view(); + REQUIRE(v() == (1 / sp(36) + 4 / sp(36) + 9 / sp(36))); + + // Test higher-order cases + result = field_z.clone(); + horiz_contraction(result, field10, field00); + REQUIRE(result.get_header().get_identifier().get_layout().tags() == + std::vector({CMP})); + REQUIRE(result.get_header().get_identifier().get_layout().dim(0) == dim1); + + result = field_y.clone(); + horiz_contraction(result, field11, field00); + REQUIRE(result.get_header().get_identifier().get_layout().tags() == + std::vector({LEV})); + REQUIRE(result.get_header().get_identifier().get_layout().dim(0) == dim2); + + result = field_w.clone(); + horiz_contraction(result, field20, field00); + REQUIRE(result.get_header().get_identifier().get_layout().tags() == + std::vector({CMP, LEV})); + REQUIRE(result.get_header().get_identifier().get_layout().dim(0) == dim1); + REQUIRE(result.get_header().get_identifier().get_layout().dim(1) == dim2); + + // Check a 3D case + field20.sync_to_host(); + auto manual_result = result.clone(); + manual_result.deep_copy(0); + manual_result.sync_to_host(); + auto v2 = field20.get_strided_view(); + auto mr = manual_result.get_strided_view(); + for(int i = 0; i < dim0; ++i) { + for(int j = 0; j < dim1; ++j) { + for(int k = 0; k < dim2; ++k) { + mr(j, k) += v00(i) * v2(i, j, k); + } + } + } + field20.sync_to_dev(); + manual_result.sync_to_dev(); + REQUIRE(views_are_equal(result, manual_result)); + } + + SECTION("vert_contraction") { + std::vector lev_tags = {LEV, ILEV}; + // iterate over lev_tags + for(auto lev_tag : lev_tags) { + using RPDF = std::uniform_real_distribution; + auto engine = setup_random_test(); + RPDF pdf(0, 1); + + int dim0 = 4; + int dim1 = 9; + // Note that parallel reduction is happening over dim2 (LEV/ILEV) + // If it is more than 3, the order of ops will lead to non-bfb results + int dim2 = lev_tag == LEV ? 3 : 3; + + // Set a weight field + FieldIdentifier f00("f", {{lev_tag}, {dim2}}, m / s, "g"); + Field field00(f00); + field00.allocate_view(); + field00.sync_to_host(); + auto v00 = field00.get_strided_view(); + for(int i = 0; i < dim2; ++i) { + // denominator is the sum of the first dim2 integers (analytically known) + v00(i) = sp(i + 1) / sp(dim2*(dim2+1)/2); + } + field00.sync_to_dev(); + + // Create (random) sample fields + FieldIdentifier fsc("f", {{}, {}}, m / s, "g"); // scalar + FieldIdentifier f10("f", {{COL, lev_tag}, {dim0, dim2}}, m / s, "g"); + FieldIdentifier f11("f", {{CMP, lev_tag}, {dim1, dim2}}, m / s, "g"); + FieldIdentifier f20("f", {{COL, CMP, lev_tag}, {dim0, dim1, dim2}}, m / s, + "g"); + Field fieldsc(fsc); + Field field10(f10); + Field field11(f11); + Field field20(f20); + fieldsc.allocate_view(); + field10.allocate_view(); + field11.allocate_view(); + field20.allocate_view(); + randomize(fieldsc, engine, pdf); + randomize(field10, engine, pdf); + randomize(field11, engine, pdf); + randomize(field20, engine, pdf); + + FieldIdentifier F_x("fx", {{COL}, {dim0}}, m / s, "g"); + FieldIdentifier F_y("fy", {{CMP}, {dim1}}, m / s, "g"); + FieldIdentifier F_z("fz", {{COL, CMP}, {dim0, dim1}}, m / s, "g"); + + Field field_x(F_x); + Field field_y(F_y); + Field field_z(F_z); + + // Test invalid inputs + REQUIRE_THROWS(vert_contraction(fieldsc, field_x, + field00)); // x not allocated yet + + field_x.allocate_view(); + field_y.allocate_view(); + field_z.allocate_view(); + + REQUIRE_THROWS(vert_contraction(fieldsc, field_y, + field_x)); // unmatching layout + REQUIRE_THROWS(vert_contraction(field_z, field11, + field11)); // wrong weight layout + + Field result; + + // Add test for invalid rank-2 weight field layout + FieldIdentifier bad_w("bad_w", {{CMP, lev_tag}, {dim1, dim2}}, m / s, "g"); + Field bad_weight(bad_w); + bad_weight.allocate_view(); + REQUIRE_THROWS(vert_contraction(result, field20, bad_weight)); + + // Add test for mismatched weight field dimensions + FieldIdentifier wrong_size_w("wrong_w", {{COL, lev_tag}, {dim0 + 1, dim2}}, + m / s, "g"); + Field wrong_weight(wrong_size_w); + wrong_weight.allocate_view(); + REQUIRE_THROWS(vert_contraction(result, field20, wrong_weight)); + + // Ensure a scalar case works + result = fieldsc.clone(); + vert_contraction(result, field00, field00); + result.sync_to_host(); + auto v = result.get_view(); + // numerator is the sum of the suqares of the first dim2 integers (analytically known) + // denominator is the sum of the first dim2 integers squared (analytically known) + REQUIRE(v() == sp(dim2*(dim2+1)*(2*dim2+1)/6) / sp((dim2*(dim2+1)/2)*(dim2*(dim2+1)/2))); + + // Test higher-order cases + result = field_x.clone(); + vert_contraction(result, field10, field00); + REQUIRE(result.get_header().get_identifier().get_layout().tags() == + std::vector({COL})); + REQUIRE(result.get_header().get_identifier().get_layout().dim(0) == dim0); + + // Check a 2D case with 1D weight + field10.sync_to_host(); + auto manual_result = result.clone(); + manual_result.deep_copy(0); + manual_result.sync_to_host(); + auto v1 = field10.get_strided_view(); + auto mr = manual_result.get_strided_view(); + for(int i = 0; i < dim0; ++i) { + for(int j = 0; j < dim2; ++j) { + mr(i) += v00(j) * v1(i, j); + } + } + field11.sync_to_dev(); + manual_result.sync_to_dev(); + REQUIRE(views_are_equal(result, manual_result)); + + result = field_y.clone(); + vert_contraction(result, field11, field00); + REQUIRE(result.get_header().get_identifier().get_layout().tags() == + std::vector({CMP})); + REQUIRE(result.get_header().get_identifier().get_layout().dim(0) == dim1); + + result = field_z.clone(); + vert_contraction(result, field20, field00); + REQUIRE(result.get_header().get_identifier().get_layout().tags() == + std::vector({COL, CMP})); + REQUIRE(result.get_header().get_identifier().get_layout().dim(0) == dim0); + REQUIRE(result.get_header().get_identifier().get_layout().dim(1) == dim1); + + // Check a 3D case with 1D weight + field20.sync_to_host(); + result.sync_to_host(); + manual_result = result.clone(); + manual_result.deep_copy(0); + manual_result.sync_to_host(); + auto v2 = field20.get_strided_view(); + auto mr2 = manual_result.get_strided_view(); + for(int i = 0; i < dim0; ++i) { + for(int j = 0; j < dim1; ++j) { + for(int k = 0; k < dim2; ++k) { + mr2(i, j) += v00(k) * v2(i, j, k); + } + } + } + field20.sync_to_dev(); + manual_result.sync_to_dev(); + REQUIRE(views_are_equal(result, manual_result)); + + // Check a 3D case with 2D weight + result = field_z.clone(); + vert_contraction(result, field20, field10); + REQUIRE(result.get_header().get_identifier().get_layout().tags() == + std::vector({COL, CMP})); + REQUIRE(result.get_header().get_identifier().get_layout().dim(0) == dim0); + REQUIRE(result.get_header().get_identifier().get_layout().dim(1) == dim1); + + field20.sync_to_host(); + result.sync_to_host(); + manual_result = result.clone(); + manual_result.deep_copy(0); + manual_result.sync_to_host(); + auto mr3 = manual_result.get_strided_view(); + for(int i = 0; i < dim0; ++i) { + for(int j = 0; j < dim1; ++j) { + for(int k = 0; k < dim2; ++k) { + mr3(i, j) += v1(i, k) * v2(i, j, k); + } + } + } + field20.sync_to_dev(); + manual_result.sync_to_dev(); + REQUIRE(views_are_equal(result, manual_result)); + } + } + SECTION ("frobenius") { auto v1 = f1.get_strided_view(); diff --git a/components/eamxx/src/share/tests/vertical_remapper_tests.cpp b/components/eamxx/src/share/tests/vertical_remapper_tests.cpp index 34351c65db8f..b4f1627a16ec 100644 --- a/components/eamxx/src/share/tests/vertical_remapper_tests.cpp +++ b/components/eamxx/src/share/tests/vertical_remapper_tests.cpp @@ -4,37 +4,46 @@ #include "share/grid/remap/coarsening_remapper.hpp" #include "share/grid/point_grid.hpp" #include "share/io/scream_scorpio_interface.hpp" +#include "share/util/scream_timing.hpp" +#include "share/field/field_utils.hpp" namespace scream { -template -typename ViewT::HostMirror -cmvc (const ViewT& v) { - auto vh = Kokkos::create_mirror_view(v); - Kokkos::deep_copy(vh,v); - return vh; -} +constexpr int vec_dim = 3; +constexpr auto P0 = VerticalRemapper::P0; +constexpr auto Mask = VerticalRemapper::Mask; +constexpr auto Top = VerticalRemapper::Top; +constexpr auto Bot = VerticalRemapper::Bot; +constexpr auto TopBot = VerticalRemapper::TopAndBot; +constexpr Real mask_val = -99999.0; -void print (const std::string& msg, const ekat::Comm& comm) { +template +void print (const std::string& fmt, const ekat::Comm& comm, Args&&... args) { + if (comm.am_i_root()) { + printf(fmt.c_str(),std::forward(args)...); + } +} +// Overload for when there are no additional arguments +void print(const std::string& fmt, const ekat::Comm& comm) { if (comm.am_i_root()) { - printf("%s",msg.c_str()); + printf(fmt.c_str()); } } // Helper function to create a grid given the number of dof's and a comm group. std::shared_ptr -build_src_grid(const ekat::Comm& comm, const int nldofs_src, const int nlevs_src) +build_grid(const ekat::Comm& comm, const int nldofs, const int nlevs) { using gid_type = AbstractGrid::gid_type; - auto src_grid = std::make_shared("src",nldofs_src,nlevs_src,comm); + auto grid = std::make_shared("src",nldofs,nlevs,comm); - auto src_dofs = src_grid->get_dofs_gids(); - auto src_dofs_h = src_dofs.get_view(); - std::iota(src_dofs_h.data(),src_dofs_h.data()+nldofs_src,nldofs_src*comm.rank()); - src_dofs.sync_to_dev(); + auto dofs = grid->get_dofs_gids(); + auto dofs_h = dofs.get_view(); + std::iota(dofs_h.data(),dofs_h.data()+nldofs,nldofs*comm.rank()); + dofs.sync_to_dev(); - return src_grid; + return grid; } // Helper function to create fields @@ -64,7 +73,133 @@ Real data_func(const int col, const int vec, const Real pres) { // - pres, the current pressure // Should ensure that the interpolated values match exactly, since vertical interp is also a linear interpolator. // Note, we don't use the level, because here the vertical interpolation is over pressure, so it represents the level. - return col*pres + vec*100.0; + return (col+1)*pres + vec*100.0; +} + +void compute_field (const Field& f, const Field& p) +{ + Field::view_host_t p1d; + Field::view_host_t p2d; + bool rank1 = p.rank()==1; + const auto& l = f.get_header().get_identifier().get_layout(); + const int ncols = l.dims().front(); + const int nlevs = l.dims().back(); + if (rank1) { + p1d = p.get_view(); + } else { + p2d = p.get_view(); + } + + // Grab correct pressure (1d or 2d) + auto pval = [&](int i, int k) { + if (rank1) return p1d(k); + else return p2d(i,k); + }; + + switch (l.type()) { + case LayoutType::Scalar2D: + { + const auto v = f.get_view(); + for (int i=0; i(); + for (int i=0; i(); + for (int i=0; i(); + for (int i=0; i p1d_src,p1d_tgt; + Field::view_host_t p2d_src,p2d_tgt; + if (p_src.rank()==1) { + p1d_src = p_src.get_view(); + } else { + p2d_src = p_src.get_view(); + } + if (p_tgt.rank()==1) { + p1d_tgt = p_tgt.get_view(); + } else { + p2d_tgt = p_tgt.get_view(); + } + + auto pval = [&](auto p1d, auto p2d, int i, int k, int rank) { + if (rank==1) return p1d(k); + else return p2d(i,k); + }; + + const auto& l = f.get_header().get_identifier().get_layout(); + const int ncols = l.dims().front(); + const int nlevs = l.dims().back(); + const int nlevs_src = p_src.get_header().get_identifier().get_layout().dims().back(); + // print_field_hyperslab(p_src); + switch (l.type()) { + case LayoutType::Scalar2D: break; + case LayoutType::Vector2D: break; + case LayoutType::Scalar3D: + { + const auto v = f.get_view(); + for (int i=0; ipmax) { + v(i,j) = etype_bot==Mask ? mask_val : data_func(i,0,pmax); + } else if (p(); + for (int i=0; ipmax) { + v(i,j,k) = etype_bot==Mask ? mask_val : data_func(i,j,pmax); + } else if (p dofs_p(nlevs_tgt); std::iota(dofs_p.begin(),dofs_p.end(),0); std::vector p_tgt; - for (int ii=0; ii creating map file ... done!\n",comm); // -------------------------------------- // - // Build src grid and remapper // + // Build src grid // // -------------------------------------- // - print (" -> creating grid and remapper ...\n",comm); + print (" -> creating src grid ...\n",comm); + auto src_grid = build_grid(comm, nldofs, nlevs_src); + print (" -> creating src grid ... done!\n",comm); - const Real mask_val = -99999.0; + // -------------------------------------- // + // Retrieve tgt grid // + // -------------------------------------- // - auto src_grid = build_src_grid(comm, nldofs_src, nlevs_src); - - // We need the source pressure level fields for both p_mid and p_int - auto pmid_src = create_field("p_mid", src_grid, false, false, true, SCREAM_PACK_SIZE); - auto pint_src = create_field("p_int", src_grid, false, false, false, SCREAM_PACK_SIZE); - // Set the source pressures - { - // By adding 1 to the pbot_tgt and subtrating 1 from ptop_tgt we ensure some masking, which - // we also want to check. - const Real ptop_src = ptop_tgt-1; - const Real pbot_src = pbot_tgt+1; - const Real dp_src = (pbot_src-ptop_src)/(nlevs_src-1); - auto pmid_v = pmid_src.get_view(); - auto pint_v = pint_src.get_view(); - for (int ii=0; ii(src_grid,filename,pmid_src,pint_src,mask_val); - print (" -> creating grid and remapper ... done!\n",comm); + print (" -> retreiving tgt grid ...\n",comm); + auto tgt_grid = VerticalRemapper::create_tgt_grid (src_grid,filename); + print (" -> retreiving tgt grid ... done!\n",comm); // -------------------------------------- // - // Create src/tgt grid fields // + // Check tgt grid // // -------------------------------------- // - print (" -> creating fields ...\n",comm); - constexpr int vec_dim = 3; + print (" -> checking tgt grid ...\n",comm); + REQUIRE (tgt_grid->get_num_local_dofs()==src_grid->get_num_local_dofs()); + REQUIRE (tgt_grid->get_num_vertical_levels()==nlevs_tgt); + REQUIRE (tgt_grid->has_geometry_data("p_levs")); + auto p_levs = tgt_grid->get_geometry_data("p_levs"); + auto p_levs_v = p_levs.get_view(); + for (int k=0; kget_tgt_grid(); - // Check that the target grid made by the remapper has the same number of columns as the source grid. - // Also check that the number of levels matches the expectation. - REQUIRE(tgt_grid->get_num_vertical_levels()==nlevs_tgt); - REQUIRE(tgt_grid->get_num_global_dofs()==src_grid->get_num_global_dofs()); - - auto src_s2d = create_field("s2d", src_grid,true,false); - auto src_v2d = create_field("v2d", src_grid,true,true); - // For now we can only support PS = SCREAM_PACK_SIZE because the source and target pressure levels will assume that packsize. - // If we use a smaller packsize we throw an error in the property check step of the vertical interpolation scheme. - // TODO: Fix that. - auto src_s3d_m = create_field("s3d_m",src_grid,false,false,true, 1); - auto src_s3d_i = create_field("s3d_i",src_grid,false,false,false,SCREAM_PACK_SIZE); - auto src_v3d_m = create_field("v3d_m",src_grid,false,true ,true, 1); - auto src_v3d_i = create_field("v3d_i",src_grid,false,true ,false,SCREAM_PACK_SIZE); - - auto tgt_s2d = create_field("s2d", tgt_grid,true,false); - auto tgt_v2d = create_field("v2d", tgt_grid,true,true); - auto tgt_s3d_m = create_field("s3d_m",tgt_grid,false,false,true, 1); - auto tgt_s3d_i = create_field("s3d_i",tgt_grid,false,false,true, SCREAM_PACK_SIZE); - auto tgt_v3d_m = create_field("v3d_m",tgt_grid,false,true ,true, 1); - auto tgt_v3d_i = create_field("v3d_i",tgt_grid,false,true ,true, SCREAM_PACK_SIZE); - - std::vector src_f = {src_s2d,src_v2d,src_s3d_m,src_s3d_i,src_v3d_m,src_v3d_i}; - std::vector tgt_f = {tgt_s2d,tgt_v2d,tgt_s3d_m,tgt_s3d_i,tgt_v3d_m,tgt_v3d_i}; - - print (" -> creating fields ... done!\n",comm); + print (" -> checking tgt grid ... done!\n",comm); - // -------------------------------------- // - // Register fields in the remapper // - // -------------------------------------- // + // Clean up scorpio stuff + scorpio::finalize_subsystem(); - print (" -> registering fields ...\n",comm); - remap->registration_begins(); - remap->register_field(src_s2d, tgt_s2d); - remap->register_field(src_v2d, tgt_v2d); - remap->register_field(src_s3d_m,tgt_s3d_m); - remap->register_field(src_s3d_i,tgt_s3d_i); - remap->register_field(src_v3d_m,tgt_v3d_m); - remap->register_field(src_v3d_i,tgt_v3d_i); - remap->registration_ends(); - print (" -> registering fields ... done!\n",comm); + print ("Testing retrieval of tgt grid from map file ...\n",comm); +} +TEST_CASE ("vertical_remapper") { // -------------------------------------- // - // Check remapper internals // + // Init MPI and PIO // // -------------------------------------- // - print (" -> Checking remapper internal state ...\n",comm); + ekat::Comm comm(MPI_COMM_WORLD); + print ("Testing vertical remapper ...\n",comm); - // Check the pressure levels read from map file + scorpio::init_subsystem(comm); // -------------------------------------- // - // Generate data for src fields // + // Set grid/map sizes // // -------------------------------------- // - print (" -> generate src fields data ...\n",comm); - using namespace ShortFieldTagsNames; - // Generate data in a deterministic way, so that when we check results, - // we know a priori what the input data that generated the tgt field's - // values was, even if that data was off rank. - auto pmid_v = pmid_src.get_view(); - auto pint_v = pint_src.get_view(); - auto src_gids = remap->get_src_grid()->get_dofs_gids().get_view(); - for (const auto& f : src_f) { - const auto& l = f.get_header().get_identifier().get_layout(); - switch (l.type()) { - case LayoutType::Scalar2D: - { - const auto v_src = f.get_view(); - for (int i=0; i creating src grid ...\n",comm); + auto src_grid = build_grid(comm, nldofs, nlevs_src); + print (" nlevs src: %d\n",comm,nlevs_src); + print (" -> creating src grid ...done!\n",comm); + + // Tgt grid must have same 2d layout as src grid + REQUIRE_THROWS (std::make_shared(src_grid,build_grid(comm,nldofs+1,nlevs_src))); + + // Helper lambda, to create p_int profile. If it is a 3d field, make same profile on each col + auto create_pint = [&](const auto& grid, const bool one_d, const Real ptop, const Real pbot) { + auto layout = one_d ? grid->get_vertical_layout(false) + : grid->get_3d_scalar_layout(false); + FieldIdentifier fid("p_int",layout,ekat::units::Pa,grid->name()); + Field pint (fid); + pint.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); + pint.allocate_view(); + + int nlevs = grid->get_num_vertical_levels(); + const Real dp = (pbot-ptop)/nlevs; + + if (one_d) { + auto pv = pint.get_view(); + pv(nlevs) = pbot; + for (int k=nlevs; k>0; --k) { + pv(k-1) = pv(k) - dp; + } + } else { + auto pv = pint.get_view(); + for (int i=0; i0; --k) { + pv(i,k-1) = pv(i,k) - dp; } - } break; - case LayoutType::Vector2D: - { - const auto v_src = f.get_view(); - for (int i=0; i(); - for (int i=0; i(); - for (int i=0; i generate src fields data ... done!\n",comm); - - // No bwd remap - REQUIRE_THROWS(remap->remap(false)); - - for (int irun=0; irun<5; ++irun) { - print (" -> run remap ...\n",comm); - remap->remap(true); - print (" -> run remap ... done!\n",comm); - - // -------------------------------------- // - // Check remapped fields // - // -------------------------------------- // - - print (" -> check tgt fields ...\n",comm); - const auto tgt_gids = tgt_grid->get_dofs_gids().get_view(); - const int ntgt_gids = tgt_gids.size(); - for (size_t ifield=0; ifield Checking field with source layout " + ls +" " + dots + "\n",comm); - - f.sync_to_host(); - - switch (lsrc.type()) { - case LayoutType::Scalar2D: - { - // This is a flat array w/ no LEV tag so the interpolated value for source and target should match. - const auto v_src = fsrc.get_view(); - const auto v_tgt = f.get_view(); - for (int i=0; i(); - const auto v_src = fsrc.get_view(); - for (int i=0; i(); - for (int i=0; ip_v(i,nlevs_p-1) || p_tgt[j](); - for (int i=0; ip_v(i,nlevs_p-1) || p_tgt[k](); + auto pmid_v = pmid.get_view< Real*,Host>(); + for (int k=0; k(); + auto pmid_v = pmid.get_view< Real**,Host>(); + for (int i=0; i Checking field with source layout " + ls + " " + dots + " OK!\n",comm); + // Test tgt grid with 2x and 0.5x as many levels as src grid + for (int nlevs_tgt : {nlevs_src/2, 2*nlevs_src}) { + for (bool src_1d : {true, false}) { + for (bool tgt_1d : {true, false}) { + for (auto etype_top : {P0, Mask}) { + for (auto etype_bot : {P0, Mask}) { + print ("************************************************\n",comm); + print (" nlevs tgt: %d\n",comm,nlevs_tgt); + print (" src pressure is 1d: %s\n",comm,src_1d ? "true" : "false"); + print (" tgt pressure is 1d: %s\n",comm,tgt_1d ? "true" : "false"); + print (" extrap type at top: %s\n",comm,etype_top==P0 ? "p0" : "masked"); + print (" extrap type at bot: %s\n",comm,etype_bot==P0 ? "p0" : "masked"); + print ("************************************************\n",comm); + + print (" -> creating tgt grid ...\n",comm); + auto tgt_grid = src_grid->clone("tgt",true); + tgt_grid->reset_num_vertical_lev(nlevs_tgt); + print (" -> creating tgt grid ...done!\n",comm); + + print (" -> creating src/tgt pressure fields ...\n",comm); + auto pint_src = create_pint(src_grid, src_1d, ptop_src, pbot_src); + auto pmid_src = create_pmid(pint_src); + + // Make ptop_tgtpsurf_src, so we do have extrapolation + const Real ptop_tgt = 10; + const Real pbot_tgt = 1020; + auto pint_tgt = create_pint(tgt_grid, tgt_1d, ptop_tgt, pbot_tgt); + auto pmid_tgt = create_pmid(pint_tgt); + print (" -> creating src/tgt pressure fields ... done!\n",comm); + + print (" -> creating fields ... done!\n",comm); + auto src_s2d = create_field("s2d", src_grid,true,false); + auto src_v2d = create_field("v2d", src_grid,true,true); + auto src_s3d_m = create_field("s3d_m",src_grid,false,false,true, 1); + auto src_s3d_i = create_field("s3d_i",src_grid,false,false,false,SCREAM_PACK_SIZE); + auto src_v3d_m = create_field("v3d_m",src_grid,false,true ,true, 1); + auto src_v3d_i = create_field("v3d_i",src_grid,false,true ,false,SCREAM_PACK_SIZE); + + auto tgt_s2d = create_field("s2d", tgt_grid,true,false); + auto tgt_v2d = create_field("v2d", tgt_grid,true,true); + auto tgt_s3d_m = create_field("s3d_m",tgt_grid,false,false,true, 1); + auto tgt_s3d_i = create_field("s3d_i",tgt_grid,false,false,true, SCREAM_PACK_SIZE); + auto tgt_v3d_m = create_field("v3d_m",tgt_grid,false,true ,true, 1); + auto tgt_v3d_i = create_field("v3d_i",tgt_grid,false,true ,true, SCREAM_PACK_SIZE); + + auto expected_s2d = tgt_s2d.clone(); + auto expected_v2d = tgt_v2d.clone(); + auto expected_s3d_m = tgt_s3d_m.clone(); + auto expected_s3d_i = tgt_s3d_i.clone(); + auto expected_v3d_m = tgt_v3d_m.clone(); + auto expected_v3d_i = tgt_v3d_i.clone(); + print (" -> creating fields ... done!\n",comm); + + // -------------------------------------- // + // Register fields in the remapper // + // -------------------------------------- // + + print (" -> creating and initializing remapper ...\n",comm); + auto remap = std::make_shared(src_grid,tgt_grid); + remap->set_source_pressure (pmid_src, pint_src); + remap->set_target_pressure (pmid_tgt, pint_tgt); + remap->set_extrapolation_type(etype_top,Top); + remap->set_extrapolation_type(etype_bot,Bot); + REQUIRE_THROWS (remap->set_mask_value(std::numeric_limits::quiet_NaN())); + remap->set_mask_value(mask_val); // Only needed if top and/or bot use etype=Mask + + remap->registration_begins(); + remap->register_field(src_s2d, tgt_s2d); + remap->register_field(src_v2d, tgt_v2d); + remap->register_field(src_s3d_m,tgt_s3d_m); + remap->register_field(src_s3d_i,tgt_s3d_i); + remap->register_field(src_v3d_m,tgt_v3d_m); + remap->register_field(src_v3d_i,tgt_v3d_i); + remap->registration_ends(); + print (" -> creating and initializing remapper ... done!\n",comm); + + // -------------------------------------- // + // Generate data for src fields // + // -------------------------------------- // + + print (" -> generate fields data ...\n",comm); + compute_field(src_s2d, pmid_src); + compute_field(src_v2d, pmid_src); + compute_field(src_s3d_m,pmid_src); + compute_field(src_s3d_i,pint_src); + compute_field(src_v3d_m,pmid_src); + compute_field(src_v3d_i,pint_src); + + // Pre-compute what we expect the tgt fields to be + compute_field(expected_s2d, pmid_tgt); + compute_field(expected_v2d, pmid_tgt); + compute_field(expected_s3d_m,pmid_tgt); + compute_field(expected_s3d_i,pint_tgt); + compute_field(expected_v3d_m,pmid_tgt); + compute_field(expected_v3d_i,pint_tgt); + + extrapolate(pmid_src,pmid_tgt,expected_s2d, etype_top,etype_bot); + extrapolate(pmid_src,pmid_tgt,expected_v2d, etype_top,etype_bot); + extrapolate(pmid_src,pmid_tgt,expected_s3d_m,etype_top,etype_bot); + extrapolate(pint_src,pint_tgt,expected_s3d_i,etype_top,etype_bot); + extrapolate(pmid_src,pmid_tgt,expected_v3d_m,etype_top,etype_bot); + extrapolate(pint_src,pint_tgt,expected_v3d_i,etype_top,etype_bot); + print (" -> generate fields data ... done!\n",comm); + + // -------------------------------------- // + // Perform remap // + // -------------------------------------- // + + // No bwd remap + REQUIRE_THROWS(remap->remap(false)); + + print (" -> run remap ...\n",comm); + remap->remap(true); + print (" -> run remap ... done!\n",comm); + + // -------------------------------------- // + // Check remapped fields // + // -------------------------------------- // + + using namespace Catch::Matchers; + Real tol = 10*std::numeric_limits::epsilon(); + + print (" -> check tgt fields ...\n",comm); + { + auto diff = tgt_s2d.clone("diff"); + auto ex_norm = frobenius_norm(expected_s2d); + diff.update(expected_s2d,1/ex_norm,-1/ex_norm); + REQUIRE (frobenius_norm(diff)(expected_v2d); + diff.update(expected_v2d,1/ex_norm,-1/ex_norm); + REQUIRE (frobenius_norm(diff)(expected_s3d_m); + diff.update(expected_s3d_m,1/ex_norm,-1/ex_norm); + REQUIRE (frobenius_norm(diff)(expected_s3d_i); + diff.update(expected_s3d_i,1/ex_norm,-1/ex_norm); + REQUIRE (frobenius_norm(diff)(expected_v3d_m); + diff.update(expected_v3d_m,1 / ex_norm,-1 / ex_norm); + REQUIRE (frobenius_norm(diff)(expected_v3d_i); + diff.update(expected_v3d_i,1 / ex_norm,-1 / ex_norm); + REQUIRE (frobenius_norm(diff) check tgt fields ... done!\n",comm); + } + } + } } - print ("check tgt fields ... done!\n",comm); } // Clean up scorpio stuff scorpio::finalize_subsystem(); + + print ("Testing vertical remapper ... done!\n",comm); } } // namespace scream diff --git a/components/eamxx/src/share/util/eamxx_ad_test.cpp b/components/eamxx/src/share/util/eamxx_ad_test.cpp index 0f64bd7c5ab3..ba4934788782 100644 --- a/components/eamxx/src/share/util/eamxx_ad_test.cpp +++ b/components/eamxx/src/share/util/eamxx_ad_test.cpp @@ -32,7 +32,7 @@ TEST_CASE("scream_ad_test") { // Create a comm ekat::Comm atm_comm (MPI_COMM_WORLD); - // User can prescribe input file name via --ekat-test-params ifile= + // User can prescribe input file name via --args --ifile auto& session = ekat::TestSession::get(); session.params.emplace("ifile","input.yaml"); std::string fname = session.params["ifile"]; diff --git a/components/eamxx/src/share/util/eamxx_data_interpolation.cpp b/components/eamxx/src/share/util/eamxx_data_interpolation.cpp new file mode 100644 index 000000000000..92fec65151b7 --- /dev/null +++ b/components/eamxx/src/share/util/eamxx_data_interpolation.cpp @@ -0,0 +1,455 @@ +#include "share/util/eamxx_data_interpolation.hpp" + +#include "share/grid/remap/identity_remapper.hpp" +#include "share/grid/remap/vertical_remapper.hpp" +#include "share/grid/remap/refining_remapper_p2p.hpp" +#include "share/io/scream_scorpio_interface.hpp" +#include "share/io/scream_io_utils.hpp" +#include "share/util/scream_universal_constants.hpp" + +#include +#include +#include +#include + +namespace scream{ + +DataInterpolation:: +DataInterpolation (const std::shared_ptr& model_grid, + const std::vector& fields) + : m_model_grid (model_grid) + , m_fields (fields) +{ + EKAT_REQUIRE_MSG (model_grid!=nullptr, + "[DataInterpolation] Error! Invalid grid pointer.\n"); + + m_nfields = m_fields.size(); + m_comm = model_grid->get_comm(); +} + +void DataInterpolation::run (const util::TimeStamp& ts) +{ + EKAT_REQUIRE_MSG (m_data_initialized, + "[DataInterpolation] Error! You must call 'init_data_interval' before calling 'run'.\n"); + + // If we went past the current interval end, we need to update the end state + if (not m_data_interval.contains(ts)) { + shift_data_interval (); + } + + // Perform the time interpolation: f_out = f_beg*alpha + f_end*(1-alpha), + // where alpha = (ts-t_beg) / (t_end-t_beg). + // NOTE: pay attention to time strategy, since for YearlyPeriodic you may + // have t_beg>t_end + util::TimeInterval beg_to_ts (m_data_interval.beg,ts,m_data_interval.timeline); + double alpha = beg_to_ts.length / m_data_interval.length; + EKAT_REQUIRE_MSG (alpha>=0 and alpha<=1, + "[DataInterpolation] Error! Input timestamp is outside the current data time interval.\n" + " data interval beg ; " + m_data_interval.beg.to_string() + "\n" + " data interval end ; " + m_data_interval.end.to_string() + "\n" + " input timestamp ; " + ts.to_string() + "\n" + " interval length : " + std::to_string(m_data_interval.length) + "\n" + " interpolation coeff: " + std::to_string(alpha) + "\n"); + + for (int i=0; iget_tgt_field(i); + const auto& end = m_horiz_remapper_end->get_tgt_field(i); + auto out = m_vert_remapper->get_src_field(i); + + out.deep_copy(beg); + out.update(end,alpha,1-alpha); + } + // For Dynamic3D profile we also need to compute the source pressure profile + // NOTE: this can't be done in the loop above, since src_p is not a "remapped" + // field in the vertical remapper (also, we need to use ad different ptr) + if (m_vr_type==Dynamic3D) { + // The pressure field is THE LAST registered in the horiz remappers + const auto p_beg = m_horiz_remapper_beg->get_tgt_field(m_nfields); + const auto p_end = m_horiz_remapper_end->get_tgt_field(m_nfields); + + auto p = m_vremap->get_source_pressure(true); // mid or int doesn't matter + p.deep_copy(p_beg); + p.update(p_end,alpha,1-alpha); + } + + m_vert_remapper->remap(true); +} + +void DataInterpolation::shift_data_interval () +{ + m_curr_interval_idx.first = m_curr_interval_idx.second; + m_curr_interval_idx.second = m_time_database.get_next_idx(m_curr_interval_idx.first); + + m_data_interval.advance(m_time_database.slices[m_curr_interval_idx.second].time); + std::swap (m_horiz_remapper_beg,m_horiz_remapper_end); + update_end_fields (); +} + +void DataInterpolation:: +update_end_fields () +{ + // First, set the correct fields in the reader + std::vector fields; + for (int i=0; iget_src_field(i)); + } + + if (m_vr_type==Dynamic3D) { + // We also need to read the src pressure profile + fields.push_back(m_horiz_remapper_end->get_src_field(m_nfields)); + } + m_reader->set_fields(fields); + + // If we're also changing the file, must (re)init the scorpio structures + const auto& slice = m_time_database.slices[m_curr_interval_idx.second]; + if (m_reader->get_filename()!=slice.filename) { + m_reader->reset_filename(slice.filename); + } + + // Read and interpolate fields + m_reader->read_variables(slice.time_idx); + m_horiz_remapper_end->remap(true); +} + +void DataInterpolation:: +init_data_interval (const util::TimeStamp& t0) +{ + EKAT_REQUIRE_MSG (m_remappers_created, + "[DataInterpolation] Error! Cannot call 'init_data_interval' until after remappers creation.\n"); + + // Create a bare reader. Fields and filename are set inside the update_end_fields call + strvec_t fnames; + for (auto f : m_fields) { + fnames.push_back(f.name()); + } + + m_reader = std::make_shared(fnames,m_horiz_remapper_beg->get_src_grid()); + + // Loop over all stored time slices to find an interval that contains t0 + auto t0_interval = m_time_database.find_interval(t0); + const auto& t_beg = m_time_database.slices[t0_interval].time; + + // We need to read in the beg/end fields for the initial interval. However, our generic + // framework can only load the end slice (since that's what we need at runtime). + // So, load end state for t=t_beg, then call shift_data_interval + // NOTE: don't compute length now, since beg time point is invalid (we don't need length yet). + m_data_interval = util::TimeInterval (util::TimeStamp(),t_beg,m_time_database.timeline,false); + m_curr_interval_idx.second = t0_interval; + update_end_fields (); + shift_data_interval (); + + m_data_initialized = true; +} + +void DataInterpolation:: +setup_time_database (const strvec_t& input_files, + const util::TimeLine timeline) +{ + // Log the final list of files, so the user know if something went wrong (e.g. a bad regex) + if (m_dbg_output and m_comm.am_i_root()) { + std::cout << "Setting up DataInerpolation object. List of input files:\n"; + for (const auto& fname : input_files) { + std::cout << " - " << fname << "\n"; + } + } + + // Make sure there are no repetitions + auto num_unique_files = std::unordered_set(input_files.begin(),input_files.end()).size(); + EKAT_REQUIRE_MSG (num_unique_files==input_files.size(), + "[DataInterpolation] Error! The input files list contains duplicates.\n" + " - input_files:\n " + ekat::join(input_files,"\n ") + "\n"); + + // We perform a bunch of checks on the input files + namespace fs = std::filesystem; + + auto file_readable = [] (const std::string& fileName) { + std::ifstream file(fileName); + return file.good(); // Check if the file can be opened + }; + + // Read what time stamps we have in each file + auto ts2str = [](const util::TimeStamp& t) { return t.to_string(); }; + std::vector> times; + for (const auto& fname : input_files) { + EKAT_REQUIRE_MSG (file_readable(input_files.back()), + "Error! One of the input files is not readable.\n" + " - file : " + input_files.back() + "\n"); + + scorpio::register_file(fname,scorpio::Read); + + auto file_times = scorpio::get_all_times(fname); + EKAT_REQUIRE_MSG (file_times.size()>0, + "[DataInterpolation] Error! Input file contains no time variable.\n" + " - file name: " + fname + "\n"); + + auto t_ref = read_timestamp (fname,"reference_time_stamp"); + + times.emplace_back(); + for (const auto& t : file_times) { + times.back().push_back(t_ref + t*constants::seconds_per_day); + } + scorpio::release_file(fname); + + // Ensure time slices are sorted (it would make code messy otherwise) + EKAT_REQUIRE_MSG (std::is_sorted(times.back().begin(),times.back().end()), + "[DataInterpolation] Error! One of the input files has time slices not sorted.\n" + " - file name : " + fname + "\n" + " - time stamps: " + ekat::join(times.back(),ts2str,", ") + "\n"); + } + + // Sort the files based on start date + auto fileCmp = [](const std::vector& times1, + const std::vector& times2) + { + return times1.front() < times2.front(); + }; + std::sort(times.begin(),times.end(),fileCmp); + + // Setup the time database + m_time_database.timeline = timeline; + m_time_database.files = input_files; + + int nfiles = input_files.size(); + for (int i=0; i0) { + // Ensure files don't overlap (it would be a mess) + const auto& prev = times[i-1]; + const auto& next = times[i]; + EKAT_REQUIRE_MSG (prev.back() < next.front(), + "[DataInterpolation] Error! The input files contain overlapping time slices.\n" + " - file1 name : " + input_files[i-1] + "\n" + " - file2 name : " + input_files[i] + "\n" + " - file1 times: " + ekat::join(prev,ts2str,", ") + "\n" + " - file2 times: " + ekat::join(next,ts2str,", ") + "\n"); + } + } + + // To avoid trouble in our logic of handling time stamps relationshipc, + // we must ensure we have 2+ time slices overall + EKAT_REQUIRE_MSG (m_time_database.size()>=2, + "[DataInterpolation] Error! Input file(s) only contain 1 time slice overall.\n"); + + m_time_db_created = true; +} + +void DataInterpolation:: +setup_remappers (const std::string& hremap_filename, + const VRemapType vr_type, + const std::string& data_pname, + const Field& model_pmid, + const Field& model_pint) +{ + setup_remappers(hremap_filename, + vr_type,"P0","P0", + -1, // Unused, since we do P0 extrapolation at top/bot + data_pname,model_pmid,model_pint); +} + +void DataInterpolation:: +setup_remappers (const std::string& hremap_filename, + const VRemapType vr_type, + const std::string& extrap_type_top, + const std::string& extrap_type_bot, + const Real mask_value, + const std::string& data_pname, + const Field& model_pmid, + const Field& model_pint) +{ + EKAT_REQUIRE_MSG (m_time_db_created, + "[DataInterpolation] Error! Cannot create remappers before time database.\n"); + + using IDR = IdentityRemapper; + constexpr auto SAT = IDR::SrcAliasTgt; + + // Whether horiz remap happens or not, the tgt grid of hremap is the same + // as the model grid, but with the same nubmer of levels as in the input files + auto grid_after_hremap = m_model_grid->clone("after_hremap",true); + int nlevs_data = get_input_files_dimlen ("lev"); + grid_after_hremap->reset_num_vertical_lev(nlevs_data); + + if (hremap_filename!="") { + m_horiz_remapper_beg = std::make_shared(grid_after_hremap,hremap_filename); + m_horiz_remapper_end = std::make_shared(grid_after_hremap,hremap_filename); + } else { + // If there's NO hremap, then ncols from the data must match the model grid (nlev can differ) + int ncols = get_input_files_dimlen ("ncol"); + EKAT_REQUIRE_MSG (ncols==m_model_grid->get_num_global_dofs(), + "Error! No horiz remap was requested, but the 'ncol' dim from file does not match with the model grid one.\n" + " - model grid num global cols: " + std::to_string(m_model_grid->get_num_global_dofs()) + "\n" + " - input data num global cols: " + std::to_string(ncols) + "\n"); + + m_horiz_remapper_beg = std::make_shared(grid_after_hremap,SAT); + m_horiz_remapper_end = std::make_shared(grid_after_hremap,SAT); + } + + if (vr_type!=None) { + auto s2et = [](const std::string& s) { + if (s=="P0") { + return VerticalRemapper::P0; + } else if (s=="Mask") { + return VerticalRemapper::Mask; + } else { + EKAT_ERROR_MSG ( + "Error! Invalid/unsupported extrapolation type.\n" + " - input value : " + s + "\n" + " - valid values: P0, Mask\n"); + return static_cast(-1); + } + }; + + m_vert_remapper = m_vremap = std::make_shared(grid_after_hremap,m_model_grid); + m_vremap->set_extrapolation_type(s2et(extrap_type_top),VerticalRemapper::Top); + m_vremap->set_extrapolation_type(s2et(extrap_type_bot),VerticalRemapper::Bot); + m_vremap->set_mask_value(mask_value); + } else { + // If no vert remap is requested, model_grid and grid_after_hremap MUST have same nlevs + int model_nlevs = m_model_grid->get_num_vertical_levels(); + EKAT_REQUIRE_MSG (model_nlevs==nlevs_data, + "Error! No vertical remap was requested, but the 'lev' dim from file does not match the model grid one.\n" + " - model grid num vert levels: " + std::to_string(model_nlevs) + "\n" + " - input data num vert levels: " + std::to_string(nlevs_data) + "\n"); + m_vert_remapper = std::make_shared(grid_after_hremap,SAT); + } + + // Setup vertical pressure profiles (which can add 1 extra field to hremap) + // This MUST be done before registering in vremap, since register_field_from_tgt + // REQUIRES to have source pressure profiles set BEFORE. + Field data_p; + if (vr_type==Dynamic3D) { + // We also need to load and remap the pressure from the input files + auto hr_tgt_grid = m_horiz_remapper_beg->get_tgt_grid(); + auto p_layout = hr_tgt_grid->get_3d_scalar_layout(true); + data_p = Field (FieldIdentifier(data_pname,p_layout,ekat::units::Pa,hr_tgt_grid->name())); + data_p.allocate_view(); + + m_vremap->set_source_pressure (data_p,VerticalRemapper::Both); + m_vremap->set_target_pressure (model_pmid,model_pint); + } else if (vr_type==Static1D) { + auto hr_tgt_grid = m_horiz_remapper_beg->get_tgt_grid(); + auto p_layout = hr_tgt_grid->get_vertical_layout(true); + data_p = Field (FieldIdentifier(data_pname,p_layout,ekat::units::Pa,hr_tgt_grid->name())); + data_p.allocate_view(); + + // Use raw scorpio to read this var, since it's not decomposed. Use any file, since it's static + auto filename = m_time_database.files.front(); + scorpio::register_file(filename,scorpio::Read); + scorpio::read_var(filename,data_pname,data_p.get_internal_view_data()); + scorpio::release_file(filename); + data_p.sync_to_dev(); + + m_vremap->set_source_pressure (data_p,VerticalRemapper::Both); + m_vremap->set_target_pressure (model_pmid,model_pint); + } + m_vr_type = vr_type; + + // Register fields in the remappers. Vertical first, since we only have model-grid fields + m_vert_remapper->registration_begins(); + for (int i=0; iregister_field_from_tgt(m_fields[i]); + } + m_vert_remapper->registration_ends(); + + m_horiz_remapper_beg->registration_begins(); + m_horiz_remapper_end->registration_begins(); + for (int i=0; iget_src_field(i); + m_horiz_remapper_beg->register_field_from_tgt(f.clone()); + m_horiz_remapper_end->register_field_from_tgt(f.clone()); + } + if (vr_type==Dynamic3D) { + m_horiz_remapper_beg->register_field_from_tgt(data_p.clone()); + m_horiz_remapper_end->register_field_from_tgt(data_p.clone()); + } + m_horiz_remapper_beg->registration_ends(); + m_horiz_remapper_end->registration_ends(); + + m_remappers_created = true; +} + +int DataInterpolation::TimeDatabase:: +get_next_idx (int prev) const +{ + int next = prev+1; + if (next >= size()) { + EKAT_REQUIRE_MSG (timeline==util::TimeLine::YearlyPeriodic, + "[TimeDatabase::get_next_idx] Error! Requesting slice that is past the database end.\n"); + next = next % size(); + } + return next; +} + +int DataInterpolation::TimeDatabase:: +find_interval (const util::TimeStamp& t) const +{ + EKAT_REQUIRE_MSG (size()>1, + "[TimeDatabase::find_interval] Error! The database has not been initialized yet.\n"); + + auto contains = [&](int beg, int end, const util::TimeStamp& t) { + const auto& t_beg = slices[beg].time; + const auto& t_end = slices[end].time; + util::TimeInterval t_int (t_beg,t_end,timeline); + return t_int.contains(t); + }; + int beg=0; + int end=1; + while (end; + enum VRemapType { + None, + Static1D, + Dynamic3D + }; + + // Constructor(s) & Destructor + DataInterpolation (const std::shared_ptr& model_grid, + const std::vector& fields); + + ~DataInterpolation () = default; + + void toggle_debug_output (bool enable_dbg_output) { m_dbg_output = enable_dbg_output; } + + void setup_time_database (const strvec_t& input_files, const util::TimeLine timeline); + + void setup_remappers (const std::string& hremap_filename, + const VRemapType vr_type, + const std::string& data_pname, + const Field& model_pmid, + const Field& model_pint); + + void setup_remappers (const std::string& hremap_filename, + const VRemapType vr_type, + const std::string& extrap_type_top, + const std::string& extrap_type_bot, + const Real mask_value, + const std::string& data_pname, + const Field& model_pmid, + const Field& model_pint); + + void init_data_interval (const util::TimeStamp& t0); + + void run (const util::TimeStamp& ts); + +protected: + + void shift_data_interval (); + void update_end_fields (); + + int get_input_files_dimlen (const std::string& dimname) const; + + // ----------- Internal data types ---------- // + + struct DataSlice { + util::TimeStamp time; + std::string filename; + int time_idx; // slice index within the input file + }; + + struct TimeDatabase { + strvec_t files; + std::vector slices; + util::TimeLine timeline; + + int size () const { return slices.size(); } + int get_next_idx (int prev_idx) const; + + // Find interval containing t + int find_interval (const util::TimeStamp& t) const; + }; + + // --------------- Internal data ------------- // + + std::shared_ptr m_reader; + + std::shared_ptr m_model_grid; + + std::vector m_fields; + + // Use two horiz remappers, so we only set them up once (it may be costly) + std::shared_ptr m_horiz_remapper_beg; + std::shared_ptr m_horiz_remapper_end; + std::shared_ptr m_vert_remapper; + std::shared_ptr m_vremap; + + VRemapType m_vr_type; + int m_nfields; + + util::TimeInterval m_data_interval; + std::pair m_curr_interval_idx; + + TimeDatabase m_time_database; + + ekat::Comm m_comm; + ekat::ParameterList m_params; + + bool m_time_db_created = false; + bool m_remappers_created = false; + bool m_data_initialized = false; + + bool m_dbg_output = false; +}; + +} // namespace scream + +#endif // EAMXX_DATA_INTERPOLATION_HPP diff --git a/components/eamxx/src/share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.cpp b/components/eamxx/src/share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.cpp index 7f3f7817ea81..876ab819c7d8 100644 --- a/components/eamxx/src/share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.cpp +++ b/components/eamxx/src/share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.cpp @@ -1,11 +1,5 @@ -#include "share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.hpp" - namespace scream { bool fvphyshack = false; -void fv_phys_rrtmgp_active_gases_set_restart (const bool restart) { - TraceGasesWorkaround::singleton().set_restart(restart); -} - -} +} // namespace scream diff --git a/components/eamxx/src/share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.hpp b/components/eamxx/src/share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.hpp index 68154c093388..bdfdb4cbd38f 100644 --- a/components/eamxx/src/share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.hpp +++ b/components/eamxx/src/share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.hpp @@ -1,7 +1,5 @@ #include "share/grid/remap/abstract_remapper.hpp" -#include "ekat/ekat_parameter_list.hpp" - namespace scream { // TODO [rrtmgp active gases] This is to address issue #1782. It supports option @@ -12,22 +10,19 @@ struct TraceGasesWorkaround { private: - TraceGasesWorkaround() { restart = false; } + TraceGasesWorkaround () = default; - bool restart; std::shared_ptr remapper; std::vector active_gases; // other than h2o public: + RunType run_type = RunType::Initial; static TraceGasesWorkaround& singleton() { static TraceGasesWorkaround self; return self; } - void set_restart (const bool is_restart) { - restart = is_restart; - } void set_remapper (const std::shared_ptr& remap_ptr) { remapper = remap_ptr; } @@ -36,12 +31,10 @@ struct TraceGasesWorkaround active_gases.push_back(gas_name); } - bool is_restart() const { return restart; } std::shared_ptr get_remapper() const { return remapper; } std::vector get_active_gases() const { return active_gases; } }; extern bool fvphyshack; -void fv_phys_rrtmgp_active_gases_set_restart(const bool restart); } // namespace scream diff --git a/components/eamxx/src/share/util/scream_data_type.hpp b/components/eamxx/src/share/util/scream_data_type.hpp index 31ce6b9f8ef7..8c45cd734b6f 100644 --- a/components/eamxx/src/share/util/scream_data_type.hpp +++ b/components/eamxx/src/share/util/scream_data_type.hpp @@ -52,6 +52,7 @@ inline std::string e2str (const DataType data_type) { default: EKAT_ERROR_MSG("Error! Unsupported DataType value.\n"); } + return ""; } inline int get_type_size (const DataType data_type) { @@ -62,6 +63,7 @@ inline int get_type_size (const DataType data_type) { default: EKAT_ERROR_MSG("Error! Unsupported DataType value.\n"); } + return -1; } } // namespace scream diff --git a/components/eamxx/src/share/util/scream_setup_random_test.hpp b/components/eamxx/src/share/util/scream_setup_random_test.hpp index 419bd4ee64e9..09f3f3099d88 100644 --- a/components/eamxx/src/share/util/scream_setup_random_test.hpp +++ b/components/eamxx/src/share/util/scream_setup_random_test.hpp @@ -41,9 +41,13 @@ inline int get_random_test_seed(const ekat::Comm* comm=nullptr) } template -Engine setup_random_test(const ekat::Comm* comm=nullptr) +Engine setup_random_test(const ekat::Comm* comm=nullptr, int* return_seed=nullptr) { - return Engine (get_random_test_seed(comm)); + int seed = get_random_test_seed(comm); + if (return_seed != nullptr) { + *return_seed = seed; + } + return Engine (seed); } template diff --git a/components/eamxx/src/share/util/scream_time_stamp.cpp b/components/eamxx/src/share/util/scream_time_stamp.cpp index d2ef8243e258..3bd8d092964f 100644 --- a/components/eamxx/src/share/util/scream_time_stamp.cpp +++ b/components/eamxx/src/share/util/scream_time_stamp.cpp @@ -16,15 +16,6 @@ namespace scream { namespace util { -int days_in_month (const int yy, const int mm) { - EKAT_REQUIRE_MSG (mm>=1 && mm<=12, - "Error! Month out of bounds. Did you call `days_in_month` with yy and mm swapped?\n"); - constexpr int nonleap_days [12] = {31,28,31,30,31,30,31,31,30,31,30,31}; - constexpr int leap_days [12] = {31,29,31,30,31,30,31,31,30,31,30,31}; - auto& arr = is_leap_year(yy) ? leap_days : nonleap_days; - return arr[mm-1]; -} - bool is_leap_year (const int yy) { if (use_leap_year()) { if (yy%4==0) { @@ -41,6 +32,15 @@ bool is_leap_year (const int yy) { return false; } +int days_in_month (const int yy, const int mm) { + EKAT_REQUIRE_MSG (mm>=1 && mm<=12, + "Error! Month out of bounds. Did you call `days_in_month` with yy and mm swapped?\n"); + constexpr int nonleap_days [12] = {31,28,31,30,31,30,31,31,30,31,30,31}; + constexpr int leap_days [12] = {31,29,31,30,31,30,31,31,30,31,30,31}; + auto& arr = is_leap_year(yy) ? leap_days : nonleap_days; + return arr[mm-1]; +} + TimeStamp::TimeStamp() : m_date (3,std::numeric_limits::lowest()) , m_time (3,std::numeric_limits::lowest()) @@ -135,6 +135,23 @@ double TimeStamp::frac_of_year_in_days () const { return doy; } +int TimeStamp::days_in_curr_month () const +{ + return days_in_month(m_date[0],m_date[1]); +} + +int TimeStamp::days_in_curr_year () const +{ + return is_leap_year(m_date[0]) ? 366 : 365; +} + +TimeStamp TimeStamp::curr_month_beg () const +{ + auto date = m_date; + date[2] = 1; + return TimeStamp (date,{0,0,0}); +} + TimeStamp& TimeStamp::operator+=(const double seconds) { // Sanity checks // Note: (x-int(x)) only works for x small enough that can be stored in an int, diff --git a/components/eamxx/src/share/util/scream_time_stamp.hpp b/components/eamxx/src/share/util/scream_time_stamp.hpp index 259686eacf65..1fd1c04e4215 100644 --- a/components/eamxx/src/share/util/scream_time_stamp.hpp +++ b/components/eamxx/src/share/util/scream_time_stamp.hpp @@ -45,6 +45,11 @@ class TimeStamp { std::string get_time_string () const; double frac_of_year_in_days () const; + int days_in_curr_month () const; + int days_in_curr_year () const; + + TimeStamp curr_month_beg () const; + // === Update method(s) === // // Set the counter for the number of steps. @@ -78,13 +83,89 @@ std::int64_t operator- (const TimeStamp& ts1, const TimeStamp& ts2); // Rewind time by given number of seconds TimeStamp operator- (const TimeStamp& ts, const int dt); -// Time-related free-functions -int days_in_month (const int year, const int month); -bool is_leap_year (const int year); - // If input string is not of the format YYYY-MM-DD-XXXXX, returns an invalid time stamp TimeStamp str_to_time_stamp (const std::string& s); +// An enum describing two ways to look at timestamps: +// - Linear: treat them as part of a 1d line +// - YearlyPeriodic: treat them as part of a yearly periodic orbit +// This is used in the TimeInterval class below to correctly handle time stamps differences +enum class TimeLine { + YearlyPeriodic, + Linear +}; + +/* + * Small struct to deal with time intervals + * + * The struct simply contains timestamps for [begin,end] interval, + * and allows two things: compute the interval length (in days), and check if + * a timestamp lies within the interval. + * + * When the TimeLine arg to the ctor is YearlyPeriodic, the year part of beg/end + * time points is ignored. In this case, the length of the time interval is bound + * to be in the interval [0,365] (in non-leap years) + */ +struct TimeInterval { + + TimeStamp beg; + TimeStamp end; + TimeLine timeline = TimeLine::Linear; + double length = -1; // the interval length + + TimeInterval () = default; + TimeInterval (const util::TimeStamp& b, const util::TimeStamp& e, TimeLine tl, bool do_compute_length = true) + : beg (b), end (e), timeline (tl) + { + if (do_compute_length) + compute_length (); + } + + bool contains (const util::TimeStamp& t) const { + if (timeline==TimeLine::Linear) { + // Compare the full time stamps + return beg<=t and t<=end; + } else { + // Compare the fraction of year for beg/end and t. + // Pay extra attention to the case where new year's eve + // is in [bec,end] + auto t_frac = t.frac_of_year_in_days(); + auto end_frac = end.frac_of_year_in_days(); + auto beg_frac = beg.frac_of_year_in_days(); + bool across_nye = beg.get_month()>end.get_month(); + if (not across_nye) { + return beg_frac<=t_frac and t_frac<=end_frac; + } else { + // We are either PAST beg or BEFORE end (but not both) + return t_frac>=beg_frac or t_frac<=end_frac; + } + } + } + + void compute_length () { + if (timeline==TimeLine::Linear) { + length = end.days_from(beg); + } else { + bool across_nye = beg.get_month()>end.get_month(); + auto frac_beg = beg.frac_of_year_in_days(); + auto frac_end = end.frac_of_year_in_days(); + if (across_nye) { + double year = end.days_in_curr_year(); + length = frac_end + (year - frac_beg); + } else { + length = frac_end - frac_beg; + } + } + } + + // Advance the interval, so that it now starts from the old end + void advance(const TimeStamp& new_end) { + beg = end; + end = new_end; + compute_length(); + } +}; + } // namespace util } // namespace scream diff --git a/components/eamxx/src/share/util/scream_utils.hpp b/components/eamxx/src/share/util/scream_utils.hpp index dbb315fc4b94..9577b5597bff 100644 --- a/components/eamxx/src/share/util/scream_utils.hpp +++ b/components/eamxx/src/share/util/scream_utils.hpp @@ -368,6 +368,112 @@ constexpr int eamxx_vis_swband_idx() { return 10; } +struct DefaultMetadata { + + std::string get_longname (const std::string& name) { + if (name_2_longname.count(name)>0) { + return name_2_longname.at(name); + } else { + // TODO: Do we want to print a Warning message? I'm not sure if its needed. + return name; + } + } + + std::string get_standardname (const std::string& name) { + if (name_2_standardname.count(name)>0) { + return name_2_standardname.at(name); + } else { + // TODO: Do we want to print a Warning message? I'm not sure if its needed. + return name; + } + } + + // Create map of longnames, can be added to as developers see fit. + std::map name_2_longname = { + {"lev","hybrid level at midpoints (1000*(A+B))"}, + {"ilev","hybrid level at interfaces (1000*(A+B))"}, + {"hyai","hybrid A coefficient at layer interfaces"}, + {"hybi","hybrid B coefficient at layer interfaces"}, + {"hyam","hybrid A coefficient at layer midpoints"}, + {"hybm","hybrid B coefficient at layer midpoints"} + }; + + // Create map of longnames, can be added to as developers see fit. + std::map name_2_standardname = { + {"p_mid" , "air_pressure"}, + {"p_mid_at_cldtop" , "air_pressure_at_cloud_top"}, + {"T_2m" , "air_temperature"}, + {"T_mid" , "air_temperature"}, + {"T_mid_at_cldtop" , "air_temperature_at_cloud_top"}, + {"aero_g_sw" , "asymmetry_factor_of_ambient_aerosol_particles"}, + {"pbl_height" , "atmosphere_boundary_layer_thickness"}, + {"precip_liq_surf_mass" , "atmosphere_mass_content_of_liquid_precipitation"}, + {"cldlow" , "low_type_cloud_area_fraction"}, + {"cldmed" , "medium_type_cloud_area_fraction"}, + {"cldhgh" , "high_type_cloud_area_fraction"}, + {"cldtot" , "cloud_area_fraction"}, + {"cldfrac_tot_at_cldtop" , "cloud_area_fraction"}, + {"cldfrac_tot" , "cloud_area_fraction_in_atmosphere_layer"}, + {"cldfrac_tot_for_analysis" , "cloud_area_fraction_in_atmosphere_layer"}, + {"cldfrac_rad" , "cloud_area_fraction_in_atmosphere_layer"}, + {"qi" , "cloud_ice_mixing_ratio"}, + {"qc" , "cloud_liquid_water_mixing_ratio"}, + {"U" , "eastward_wind"}, + {"eff_radius_qi" , "effective_radius_of_cloud_ice_particles"}, + {"eff_radius_qc" , "effective_radius_of_cloud_liquid_water_particles"}, + {"eff_radius_qc_at_cldtop" , "effective_radius_of_cloud_liquid_water_particles_at_liquid_water_cloud_top"}, + {"eff_radius_qr" , "effective_radius_of_cloud_rain_particles"}, + {"qv" , "humidity_mixing_ratio"}, + {"cldfrac_ice_at_cldtop" , "ice_cloud_area_fraction"}, + {"cldfrac_ice" , "ice_cloud_area_fraction_in_atmosphere_layer"}, + {"omega" , "lagrangian_tendency_of_air_pressure"}, + {"landfrac" , "land_area_fraction"}, + {"latitude" , "latitude"}, + {"cldfrac_liq_at_cldtop" , "liquid_water_cloud_area_fraction"}, + {"cldfrac_liq" , "liquid_water_cloud_area_fraction_in_atmosphere_layer"}, + {"longitude" , "longitude"}, + {"rainfrac" , "mass_fraction_of_liquid_precipitation_in_air"}, + {"V" , "northward_wind"}, + {"nc" , "number_concentration_of_cloud_liquid_water_particles_in_air"}, + {"cdnc_at_cldtop" , "number_concentration_of_cloud_liquid_water_particles_in_air_at_liquid_water_cloud_top"}, + {"ni" , "number_concentration_of_ice_crystals_in_air"}, + {"aero_tau_sw" , "optical_thickness_of_atmosphere_layer_due_to_ambient_aerosol_particles"}, + {"aero_tau_lw" , "optical_thickness_of_atmosphere_layer_due_to_ambient_aerosol_particles"}, + {"aero_ssa_sw" , "single_scattering_albedo_in_air_due_to_ambient_aerosol_particles"}, + {"sunlit" , "sunlit_binary_mask"}, + {"ps" , "surface_air_pressure"}, + {"LW_flux_dn_at_model_bot" , "surface_downwelling_longwave_flux_in_air"}, + {"SW_flux_dn_at_model_bot" , "surface_downwelling_shortwave_flux_in_air"}, + {"SW_clrsky_flux_dn_at_model_bot" , "surface_downwelling_shortwave_flux_in_air_assuming_clear_sky"}, + {"phis" , "surface_geopotential"}, + {"surf_radiative_T" , "surface_temperature"}, + {"surf_sens_flux" , "surface_upward_sensible_heat_flux"}, + {"SW_flux_dn_at_model_top" , "toa_incoming_shortwave_flux"}, + {"LW_flux_up_at_model_top" , "toa_outgoing_longwave_flux"}, + {"LW_clrsky_flux_up_at_model_top" , "toa_outgoing_longwave_flux_assuming_clear_sky"}, + {"surf_evap" , "water_evapotranspiration_flux"}, + {"AtmosphereDensity" , "air_density"}, + {"PotentialTemperature" , "air_potential_temperature"}, + {"SeaLevelPressure" , "air_pressure_at_mean_sea_level"}, + {"IceWaterPath" , "atmosphere_mass_content_of_cloud_ice"}, + {"LiqWaterPath" , "atmosphere_mass_content_of_cloud_liquid_water"}, + {"VapWaterPath" , "atmosphere_mass_content_of_water_vapor"}, + {"AerosolOpticalDepth550nm" , "atmosphere_optical_thickness_due_to_ambient_aerosol_particles"}, + {"Exner" , "dimensionless_exner_function"}, + {"z_mid" , "geopotential_height"}, + {"geopotential_mid" , "geopotential_height"}, + {"RelativeHumidity" , "relative_humidity"}, + {"surface_upward_latent_heat_flux" , "surface_upward_latent_heat_flux"}, + {"LongwaveCloudForcing" , "toa_longwave_cloud_radiative_effect"}, + {"ShortwaveCloudForcing" , "toa_shortwave_cloud_radiative_effect"}, + {"VirtualTemperature" , "virtual_temperature"}, + {"VaporFlux" , "water_evapotranspiration_flux"}, + {"wind_speed" , "wind_speed"} + }; + +}; + + } // namespace scream #endif // SCREAM_UTILS_HPP diff --git a/components/eamxx/tests/generic/fail_check/valg_fail.cpp b/components/eamxx/tests/generic/fail_check/valg_fail.cpp index ba0ba6f59b3c..5bbd694dad37 100644 --- a/components/eamxx/tests/generic/fail_check/valg_fail.cpp +++ b/components/eamxx/tests/generic/fail_check/valg_fail.cpp @@ -6,15 +6,18 @@ namespace scream { TEST_CASE("force_valgrind_err") { - bool uninit; + bool* uninit = new bool[1]; int i = 0; - if (uninit) { + if (uninit[0]) { ++i; } else { i += 4; } - REQUIRE(i < 10); + if (i<4) { + printf("less than four\n"); + } + delete uninit; } } // empty namespace diff --git a/components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt index 841b7ea00c64..226a2a92c680 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt @@ -8,13 +8,7 @@ if (SCREAM_DOUBLE_PRECISION) add_subdirectory(homme_shoc_cld_spa_p3_rrtmgp_128levels) add_subdirectory(homme_shoc_cld_spa_p3_rrtmgp_pg2_dp) if (SCREAM_ENABLE_MAM) - # Once the mam4xx aerosol microphysics AtmosphereProcess is running, the - # corresponding test here needs to be reworked with valid aerosol - # initial conditions. - #add_subdirectory(homme_mam4xx_pg2) - add_subdirectory(mam/homme_shoc_cld_p3_mam_optics_rrtmgp) - add_subdirectory(mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep) - add_subdirectory(mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav) + add_subdirectory(mam) endif() endif() endif() diff --git a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp/input.yaml index 316b43ffdf73..15b8f8572b82 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp/input.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp/input.yaml @@ -41,7 +41,7 @@ atmosphere_processes: shoc: enable_column_conservation_checks: true lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/input.yaml index 8e463b8be24c..c9f0bdaa8b85 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/input.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/input.yaml @@ -46,7 +46,7 @@ atmosphere_processes: max_total_ni: 740.0e3 shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/input.yaml index d2f51e9e1ccb..d8dc6182ae4f 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/input.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/input.yaml @@ -37,7 +37,7 @@ atmosphere_processes: shoc: check_flux_state_consistency: true lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/input.yaml index b92889cdcaa0..3fd2b41cd310 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/input.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/input.yaml @@ -37,7 +37,7 @@ atmosphere_processes: max_total_ni: 740.0e3 shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/CMakeLists.txt index f56d8beefa60..db5c8585943b 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/CMakeLists.txt @@ -11,7 +11,7 @@ CreateDynamicsLib("theta-l_kokkos" 4 72 10) set (TEST_LABELS "dynamics;driver;tms;shoc;cld;spa;p3;rrtmgp;physics;dp") CreateUnitTest(homme_shoc_cld_spa_p3_rrtmgp_pg2_dp "homme_shoc_cld_spa_p3_rrtmgp_pg2_dp.cpp" LABELS ${TEST_LABELS} - LIBS cld_fraction tms shoc spa p3 scream_rrtmgp ${dynLibName} scream_control diagnostics + LIBS cld_fraction tms shoc spa iop_forcing p3 scream_rrtmgp ${dynLibName} scream_control diagnostics MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} ) diff --git a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp.cpp b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp.cpp index 235b7f22a16b..a6fa74530093 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp.cpp +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp.cpp @@ -54,6 +54,7 @@ TEST_CASE("scream_homme_physics", "scream_homme_physics") { ad.set_params(ad_params); ad.init_scorpio (); ad.init_time_stamps (t0, t0); + ad.create_output_managers (); ad.create_atm_processes (); ad.create_grids (); ad.create_fields (); diff --git a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/input.yaml index 0950f6bfdbc2..fdd46617bd59 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/input.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/input.yaml @@ -40,7 +40,7 @@ atmosphere_processes: homme: Moisture: moist physics: - atm_procs_list: [mac_aero_mic,rrtmgp] + atm_procs_list: [iop_forcing,mac_aero_mic,rrtmgp] schedule_type: Sequential Type: Group mac_aero_mic: @@ -55,7 +55,7 @@ atmosphere_processes: shoc: check_flux_state_consistency: true lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/mam/CMakeLists.txt new file mode 100644 index 000000000000..c67d46d4ffe3 --- /dev/null +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/CMakeLists.txt @@ -0,0 +1,3 @@ +add_subdirectory(homme_shoc_cld_p3_mam_optics_rrtmgp) +add_subdirectory(homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep) +add_subdirectory(homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav) diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/CMakeLists.txt index e3c562cfcd13..43ec03b0cd68 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/CMakeLists.txt @@ -79,6 +79,7 @@ set (TEST_INPUT_FILES scream/mam4xx/physprops/ocpho_rrtmg_c20240206.nc scream/mam4xx/physprops/bcpho_rrtmg_c20240206.nc scream/mam4xx/physprops/poly_rrtmg_c20240206.nc + scream/mam4xx/drydep/ne2np4/atmsrf_ne2np4_c20241017.nc ) foreach (file IN ITEMS ${TEST_INPUT_FILES}) diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/input.yaml index 1857bc534dd2..adae9b0688ae 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/input.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/input.yaml @@ -62,7 +62,7 @@ atmosphere_processes: shoc: enable_column_conservation_checks: true lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 @@ -96,6 +96,10 @@ atmosphere_processes: rrtmgp_cloud_optics_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-sw.nc rrtmgp_cloud_optics_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-lw.nc enable_column_conservation_checks: true + mam4_drydep: + # Fractional land use file + drydep_remap_file: "" + fractional_land_use_file: ${SCREAM_DATA_DIR}/mam4xx/drydep/ne2np4/atmsrf_ne2np4_c20241017.nc grids_manager: Type: Homme diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/output.yaml index 5922c718e6e6..92c6932fd269 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/output.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/output.yaml @@ -2,7 +2,6 @@ --- filename_prefix: homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep Averaging Type: Instant -Max Snapshots Per File: 1 Fields: Physics GLL: Field Names: diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/input.yaml index e6b942364886..21791e3391e6 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/input.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/input.yaml @@ -43,7 +43,7 @@ atmosphere_processes: shoc: enable_column_conservation_checks: true lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/output.yaml index be4bb16b95d6..eb6be66284a4 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/output.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/output.yaml @@ -2,7 +2,6 @@ --- filename_prefix: homme_shoc_cld_p3_mam_optics_rrtmgp Averaging Type: Instant -Max Snapshots Per File: 1 Fields: Physics GLL: Field Names: diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml index c9b493a2958c..a95d2677fe46 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml @@ -9,7 +9,7 @@ time_stepping: number_of_steps: ${NUM_STEPS} initial_conditions: - Filename: ${SCREAM_DATA_DIR}/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} pbl_height: 25.0 phis : 0.1 @@ -49,7 +49,7 @@ atmosphere_processes: shoc: check_flux_state_consistency: true lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/output.yaml index afe8437a2cf4..a48a4eba3c80 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/output.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/output.yaml @@ -2,7 +2,6 @@ --- filename_prefix: homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav_output Averaging Type: Instant -Max Snapshots Per File: 1 Fields: Physics GLL: Field Names: @@ -104,7 +103,7 @@ Fields: - aerdepwetis - aerdepwetcw - dgnumwet - - dgncur_a + - dgnum - wetdens - qaerwat - bc_c1 diff --git a/components/eamxx/tests/multi-process/dynamics_physics/model_restart/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/CMakeLists.txt index 5645a9383a7a..a0288b454b2c 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/model_restart/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/CMakeLists.txt @@ -24,20 +24,20 @@ set (CASE_TN 2023-01-01-00060) # Create the baseline (run all 6 timsteps in a single run) CreateUnitTestFromExec(model_baseline model_restart - EXE_ARGS "--use-colour no --ekat-test-params ifile=input_baseline.yaml" + EXE_ARGS "--args -ifile=input_baseline.yaml" MPI_RANKS ${SCREAM_TEST_MAX_RANKS} FIXTURES_SETUP baseline_run) # Start a simulation, but only run half of the time steps CreateUnitTestFromExec(model_initial model_restart - EXE_ARGS "--use-colour no --ekat-test-params ifile=input_initial.yaml" + EXE_ARGS "--args -ifile=input_initial.yaml" MPI_RANKS ${SCREAM_TEST_MAX_RANKS} FIXTURES_SETUP initial_run PROPERTIES RESOURCE_LOCK rpointer_file) # Restart the simulation, and run the second half of the time steps CreateUnitTestFromExec(model_restart model_restart - EXE_ARGS "--use-colour no --ekat-test-params ifile=input_restarted.yaml" + EXE_ARGS "--args -ifile=input_restarted.yaml" MPI_RANKS ${SCREAM_TEST_MAX_RANKS} FIXTURES_REQUIRED initial_run FIXTURES_SETUP restarted_run diff --git a/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_baseline.yaml b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_baseline.yaml index c308625a6e15..b5d749bfeff4 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_baseline.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_baseline.yaml @@ -41,7 +41,7 @@ atmosphere_processes: max_total_ni: 740.0e3 shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_initial.yaml b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_initial.yaml index 77bd88dc0d7e..4ab3bdc369d8 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_initial.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_initial.yaml @@ -41,7 +41,7 @@ atmosphere_processes: do_prescribed_ccn: false shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 @@ -69,7 +69,6 @@ grids_manager: # List all the yaml files with the output parameters Scorpio: model_restart: - filename_prefix: model_restart output_control: Frequency: 30 frequency_units: nsecs diff --git a/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_restarted.yaml b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_restarted.yaml index 73e638e2ece0..345424605a74 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_restarted.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_restarted.yaml @@ -9,9 +9,6 @@ time_stepping: run_t0: 2023-01-01-00030 # YYYY-MM-DD-XXXXX case_t0: 2023-01-01-00000 # YYYY-MM-DD-XXXXX -initial_conditions: - restart_casename: model_restart - atmosphere_processes: atm_procs_list: [homme,physics] schedule_type: Sequential @@ -31,7 +28,7 @@ atmosphere_processes: do_prescribed_ccn: false shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/CMakeLists.txt index 71fe2a5f43c2..88c5f1022f66 100644 --- a/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/CMakeLists.txt @@ -23,7 +23,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output_tend.yaml ${CMAKE_CURRENT_BINARY_DIR}/output_tend_subcycled.yaml) CreateUnitTestFromExec (shoc_p3_subcycled shoc_p3 - EXE_ARGS "--use-colour no --ekat-test-params ifile=input_subcycled.yaml" + EXE_ARGS "--args -ifile=input_subcycled.yaml" FIXTURES_SETUP shoc_p3_subcycled) # Run a test without subcycling and more steps @@ -38,7 +38,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output_tend.yaml ${CMAKE_CURRENT_BINARY_DIR}/output_tend_monolithic.yaml) CreateUnitTestFromExec (shoc_p3_monolithic shoc_p3 - EXE_ARGS "--use-colour no --ekat-test-params ifile=input_monolithic.yaml" + EXE_ARGS "--args -ifile=input_monolithic.yaml" FIXTURES_SETUP shoc_p3_monolithic) # Finally, compare output of the two tests diff --git a/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/input.yaml b/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/input.yaml index 4e9f9f79adbd..b23046cf2d15 100644 --- a/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/input.yaml @@ -16,7 +16,7 @@ atmosphere_processes: max_total_ni: 740.0e3 shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml index 55c62d69aa24..48045295050b 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml @@ -23,7 +23,9 @@ atmosphere_processes: srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc - + + soil_erodibility_file: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/dst_ne2np4_c20241028.nc + marine_organics_file: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/monthly_macromolecules_0.1deg_bilinear_year01_merge_ne2np4_c20241030.nc grids_manager: Type: Mesh Free geo_data_source: IC_FILE @@ -40,6 +42,13 @@ initial_conditions: topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} pbl_height: 0.0 + dstflx : 0.0 + ocnfrac: 0.10000000000000000E+001 + sst: 0.30178553874977507E+003 + cldfrac_tot: 0.138584624960092 + horiz_winds: [-0.24988988196194634E+000, -0.23959782871450760E+000] + constituent_fluxes: 0.0 + # The parameters for I/O control Scorpio: diff --git a/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml index c26db8eb6814..7d4d355ecc66 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml @@ -57,7 +57,7 @@ Field Names: - aerdepwetis - aerdepwetcw - dgnumwet - - dgncur_a + - dgnum - wetdens - qaerwat output_control: diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml index 9c0ed4d97988..1cc9eb6eb4d9 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml @@ -21,7 +21,7 @@ atmosphere_processes: do_prescribed_ccn: false shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/input.yaml index 20d6f142c52a..9483f3790bf5 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/input.yaml @@ -24,7 +24,7 @@ atmosphere_processes: do_prescribed_ccn: false shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/input.yaml index 7ec317eda928..2a571ba96f12 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/input.yaml @@ -24,7 +24,7 @@ atmosphere_processes: do_prescribed_ccn: false shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/input.yaml index 804939a22058..d970765d2bd9 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/input.yaml @@ -21,7 +21,7 @@ atmosphere_processes: do_prescribed_ccn: false shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml index 1ae53e6c1c30..da4a83ad7b18 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml @@ -91,7 +91,7 @@ Field Names: - aerdepwetis - aerdepwetcw - dgnumwet - - dgncur_a + - dgnum - wetdens - qaerwat output_control: diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml index ae997bb3da57..c70f1b8085a2 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml @@ -21,7 +21,7 @@ atmosphere_processes: top_level_mam4xx: 6 shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/CMakeLists.txt index 9f6fb8733c2c..d64cc7eebaf3 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/CMakeLists.txt @@ -21,6 +21,13 @@ set (SHOC_MAX_DT 300) math (EXPR MAC_MIC_SUBCYCLES "(${ATM_TIME_STEP} + ${SHOC_MAX_DT} - 1) / ${SHOC_MAX_DT}") # Ensure test input files are present in the data dir +set (TEST_INPUT_FILES + scream/mam4xx/drydep/ne2np4/atmsrf_ne2np4_c20241017.nc +) +foreach (file IN ITEMS ${TEST_INPUT_FILES}) + GetInputFile(${file}) +endforeach() + GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/input.yaml index c451008adc76..3f5981403091 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/input.yaml @@ -18,7 +18,7 @@ atmosphere_processes: number_of_subcycles: ${MAC_MIC_SUBCYCLES} shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 @@ -29,6 +29,10 @@ atmosphere_processes: c_diag_3rd_mom: 7.0 Ckh: 0.1 Ckm: 0.1 + mam4_drydep: + # Fractional land use file + drydep_remap_file: "" + fractional_land_use_file: ${SCREAM_DATA_DIR}/mam4xx/drydep/ne2np4/atmsrf_ne2np4_c20241017.nc grids_manager: Type: Mesh Free diff --git a/components/eamxx/tests/multi-process/physics_only/shoc_cld_p3_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_cld_p3_rrtmgp/input.yaml index fc0a1b6bc8a9..529a075b10dc 100644 --- a/components/eamxx/tests/multi-process/physics_only/shoc_cld_p3_rrtmgp/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/shoc_cld_p3_rrtmgp/input.yaml @@ -21,7 +21,7 @@ atmosphere_processes: do_prescribed_ccn: false shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/physics_only/shoc_cld_spa_p3_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_cld_spa_p3_rrtmgp/input.yaml index 542625798f53..a98cbe30e85c 100644 --- a/components/eamxx/tests/multi-process/physics_only/shoc_cld_spa_p3_rrtmgp/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/shoc_cld_spa_p3_rrtmgp/input.yaml @@ -22,7 +22,7 @@ atmosphere_processes: max_total_ni: 740.0e3 shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/CMakeLists.txt index 62c4b6d2266e..16568dc55b3a 100644 --- a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/CMakeLists.txt @@ -27,7 +27,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output_remapped.yaml ${CMAKE_CURRENT_BINARY_DIR}/output_source_data_remapped.yaml) CreateUnitTestFromExec (shoc_p3_source shoc_p3_nudging - EXE_ARGS "--use-colour no --ekat-test-params ifile=input_source_data.yaml" + EXE_ARGS "--args -ifile=input_source_data.yaml" FIXTURES_SETUP shoc_p3_source_data FIXTURES_REQUIRED shoc_p3_create_vertical_remap_and_weights_file) @@ -41,7 +41,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input_nudging.yaml configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml ${CMAKE_CURRENT_BINARY_DIR}/output_nudged.yaml) CreateUnitTestFromExec (shoc_p3_nudged shoc_p3_nudging - EXE_ARGS "--use-colour no --ekat-test-params ifile=input_nudging.yaml" + EXE_ARGS "--args -ifile=input_nudging.yaml" FIXTURES_REQUIRED shoc_p3_source_data) # Run a test with nudging turned on using remapped source data for nudging: @@ -54,7 +54,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input_nudging.yaml configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml ${CMAKE_CURRENT_BINARY_DIR}/output_nudged_remapped.yaml) CreateUnitTestFromExec (shoc_p3_nudged_remapped shoc_p3_nudging - EXE_ARGS "--use-colour no --ekat-test-params ifile=input_nudging_remapped.yaml" + EXE_ARGS "--args -ifile=input_nudging_remapped.yaml" FIXTURES_REQUIRED shoc_p3_source_data) # Run a test with nudging using data read in glob pattern and skip vertical interpolation: @@ -67,6 +67,6 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input_nudging_glob_novert.yaml configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml ${CMAKE_CURRENT_BINARY_DIR}/output_nudged_glob_novert.yaml) CreateUnitTestFromExec (shoc_p3_nudging_glob_novert shoc_p3_nudging - EXE_ARGS "--use-colour no --ekat-test-params ifile=input_nudging_glob_novert.yaml" + EXE_ARGS "--args -ifile=input_nudging_glob_novert.yaml" FIXTURES_REQUIRED shoc_p3_source_data) diff --git a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging.yaml index 6a4c0003376e..12acd670ad91 100644 --- a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging.yaml +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging.yaml @@ -23,7 +23,7 @@ atmosphere_processes: source_pressure_file: vertical_remap.nc ## Only used in the case of STATIC_1D_VERTICAL_PROFILE shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging_glob_novert.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging_glob_novert.yaml index 35d75015e2a1..68e8841586db 100644 --- a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging_glob_novert.yaml +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging_glob_novert.yaml @@ -24,7 +24,7 @@ atmosphere_processes: source_pressure_file: vertical_remap.nc ## Only used in the case of STATIC_1D_VERTICAL_PROFILE shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_source_data.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_source_data.yaml index c0c3056312e2..c6ec0e361c65 100644 --- a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_source_data.yaml +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_source_data.yaml @@ -15,7 +15,7 @@ atmosphere_processes: max_total_ni: 720.0e3 shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/single-process/CMakeLists.txt b/components/eamxx/tests/single-process/CMakeLists.txt index 40565d94bf8e..074a30c924f5 100644 --- a/components/eamxx/tests/single-process/CMakeLists.txt +++ b/components/eamxx/tests/single-process/CMakeLists.txt @@ -22,9 +22,13 @@ if (SCREAM_ENABLE_MAM) add_subdirectory(mam/optics) add_subdirectory(mam/aci) add_subdirectory(mam/drydep) - add_subdirectory(mam/wet_scav) add_subdirectory(mam/emissions) add_subdirectory(mam/constituent_fluxes) + add_subdirectory(mam/wet_scav) + + # Currently this test produces non-deterministic output. + # Commenting it so the other PRs can test normally + # add_subdirectory(mam/aero_microphys) endif() if (SCREAM_TEST_LEVEL GREATER_EQUAL SCREAM_TEST_LEVEL_EXPERIMENTAL) add_subdirectory(zm) diff --git a/components/eamxx/tests/single-process/mam/aero_microphys/CMakeLists.txt b/components/eamxx/tests/single-process/mam/aero_microphys/CMakeLists.txt new file mode 100644 index 000000000000..6f6f8b364b8a --- /dev/null +++ b/components/eamxx/tests/single-process/mam/aero_microphys/CMakeLists.txt @@ -0,0 +1,65 @@ +include (ScreamUtils) + +set (TEST_BASE_NAME mam4_aero_microphys_standalone) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LABELS mam4_aero_microphys physics + LIBS mam + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 2.5h 24h +set (RUN_T0 2021-10-12-45000) + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Ensure test input files are present in the data dir +set (TEST_INPUT_FILES + scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + scream/mam4xx/linoz/ne2np4/linoz1850-2015_2010JPL_CMIP6_10deg_58km_ne2np4_c20240724.nc + scream/mam4xx/invariants/ne2np4/oxid_ne2np4_L26_1850-2015_c20240827.nc + scream/mam4xx/linoz/Linoz_Chlorine_Loading_CMIP6_0003-2017_c20171114.nc + scream/mam4xx/photolysis/RSF_GT200nm_v3.0_c080811.nc + scream/mam4xx/photolysis/temp_prs_GT200nm_JPL10_c130206.nc + scream/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_so2_elev_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_so4_a1_elev_ne2np4_2010_clim_c20240823.nc + scream/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_so4_a2_elev_ne2np4_2010_clim_c20240823.nc + scream/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_pom_a4_elev_ne2np4_2010_clim_c20240823.nc + scream/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_bc_a4_elev_ne2np4_2010_clim_c20240823.nc + scream/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_num_a1_elev_ne2np4_2010_clim_c20240823.nc + scream/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_num_a2_elev_ne2np4_2010_clim_c20240823.nc + scream/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_num_a4_elev_ne2np4_2010_clim_c20240823.nc + scream/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_soag_elev_ne2np4_2010_clim_c20240823.nc + scream/mam4xx/drydep/season_wes.nc +) +foreach (file IN ITEMS ${TEST_INPUT_FILES}) + GetInputFile(${file}) +endforeach() + + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) + +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS mam4_aero_microphys physics + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/single-process/mam/aero_microphys/input.yaml b/components/eamxx/tests/single-process/mam/aero_microphys/input.yaml new file mode 100644 index 000000000000..48070ee3b0d9 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/aero_microphys/input.yaml @@ -0,0 +1,75 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mam4_aero_microphys] + mam4_aero_microphys: + mam4_do_cond: true + mam4_do_newnuc: true + mam4_do_coag: true + mam4_do_rename: true + mam4_o3_tau: 172800.0 + mam4_o3_sfc: 3.0E-008 + mam4_o3_lbl: 4 + mam4_psc_T : 193.0 + mam4_linoz_ymd : 20100101 + mam4_linoz_file_name : ${SCREAM_DATA_DIR}/mam4xx/linoz/ne2np4/linoz1850-2015_2010JPL_CMIP6_10deg_58km_ne2np4_c20240724.nc + mam4_oxid_file_name : ${SCREAM_DATA_DIR}/mam4xx/invariants/ne2np4/oxid_ne2np4_L26_1850-2015_c20240827.nc + mam4_oxid_ymd : 20150101 + mam4_linoz_chlorine_file : ${SCREAM_DATA_DIR}/mam4xx/linoz/Linoz_Chlorine_Loading_CMIP6_0003-2017_c20171114.nc + mam4_chlorine_loading_ymd : 20100101 + mam4_rsf_file : ${SCREAM_DATA_DIR}/mam4xx/photolysis/RSF_GT200nm_v3.0_c080811.nc + mam4_xs_long_file : ${SCREAM_DATA_DIR}/mam4xx/photolysis/temp_prs_GT200nm_JPL10_c130206.nc + elevated_emiss_ymd : 20100101 + mam4_so2_elevated_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_so2_elev_ne2np4_2010_clim_c20240726.nc + mam4_so4_a1_elevated_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_so4_a1_elev_ne2np4_2010_clim_c20240823.nc + mam4_so4_a2_elevated_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_so4_a2_elev_ne2np4_2010_clim_c20240823.nc + mam4_pom_a4_elevated_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_pom_a4_elev_ne2np4_2010_clim_c20240823.nc + mam4_bc_a4_elevated_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_bc_a4_elev_ne2np4_2010_clim_c20240823.nc + mam4_num_a1_elevated_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_num_a1_elev_ne2np4_2010_clim_c20240823.nc + mam4_num_a2_elevated_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_num_a2_elev_ne2np4_2010_clim_c20240823.nc + mam4_num_a4_elevated_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_num_a4_elev_ne2np4_2010_clim_c20240823.nc + mam4_soag_elevated_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_soag_elev_ne2np4_2010_clim_c20240823.nc + mam4_season_wes_file : ${SCREAM_DATA_DIR}/mam4xx/drydep/season_wes.nc +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + type: point_grid + aliases: [Physics] + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + phis : 1.0 + pbl_height: 1.0 + #These should come from the input file + dgnum: [1.246662106183775E-007, 4.081134799487888E-008, 1.103139143795796E-006, 1.000000011686097E-007] + dgnumwet: [2.367209731605067E-007, 6.780643470563889E-008, 3.028011448344027E-006, 1.000000096285154E-007] + wetdens: [1038.67760516297, 1046.20002003441, 1031.74623165457, 1086.79731859184] + nevapr: 0.0 + precip_total_tend: 0.0 + surf_radiative_T: 288.0 + ps: 105000.0 + horiz_winds: [-0.24988988196194634E+000, -0.23959782871450760E+000] + precip_liq_surf_mass: 0.1 + precip_ice_surf_mass: 0.1 + snow_depth_land: 0.01 + fraction_landuse: 0.0 + SW_flux_dn: 500.0 + constituent_fluxes: 0.0 +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/single-process/mam/aero_microphys/output.yaml b/components/eamxx/tests/single-process/mam/aero_microphys/output.yaml new file mode 100644 index 000000000000..5af90cac5479 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/aero_microphys/output.yaml @@ -0,0 +1,74 @@ +%YAML 1.1 +--- +filename_prefix: mam4_aero_microphys_standalone_output +Averaging Type: Instant +Fields: + Physics: + Field Names: + - T_mid + - O3 + - H2O2 + - H2SO4 + - SO2 + - DMS + - SOAG + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - so4_c1 + - so4_c2 + - so4_c3 + - pom_c1 + - pom_c3 + - pom_c4 + - soa_c1 + - soa_c2 + - soa_c3 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - num_c1 + - num_c2 + - num_c3 + - num_c4 + # To save these fields make sure to turn on + # OUTPUT_TRACER_FIELDS + #- oxi_fields + #- linoz_fields + #- vertical_emission_fields + +output_control: + Frequency: 1 + frequency_units: nsteps +... diff --git a/components/eamxx/tests/single-process/mam/drydep/CMakeLists.txt b/components/eamxx/tests/single-process/mam/drydep/CMakeLists.txt index d05e6823642f..413e6be630c4 100644 --- a/components/eamxx/tests/single-process/mam/drydep/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/drydep/CMakeLists.txt @@ -23,6 +23,13 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) # Ensure test input files are present in the data dir +set (TEST_INPUT_FILES + scream/mam4xx/drydep/ne2np4/atmsrf_ne2np4_c20241017.nc +) +foreach (file IN ITEMS ${TEST_INPUT_FILES}) + GetInputFile(${file}) +endforeach() + GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) diff --git a/components/eamxx/tests/single-process/mam/drydep/input.yaml b/components/eamxx/tests/single-process/mam/drydep/input.yaml index 86c6dd8766a2..4ae00b202978 100644 --- a/components/eamxx/tests/single-process/mam/drydep/input.yaml +++ b/components/eamxx/tests/single-process/mam/drydep/input.yaml @@ -10,6 +10,10 @@ time_stepping: atmosphere_processes: atm_procs_list: [mam4_drydep] + mam4_drydep: + # Fractional land use file + drydep_remap_file: "" + fractional_land_use_file: ${SCREAM_DATA_DIR}/mam4xx/drydep/ne2np4/atmsrf_ne2np4_c20241017.nc grids_manager: Type: Mesh Free diff --git a/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt b/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt index c8e690b929fc..65f377b4db1f 100644 --- a/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt @@ -36,6 +36,8 @@ set (TEST_INPUT_FILES scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/dst_ne2np4_c20241028.nc + scream/mam4xx/emissions/ne2np4/monthly_macromolecules_0.1deg_bilinear_year01_merge_ne2np4_c20241030.nc ) foreach (file IN ITEMS ${TEST_INPUT_FILES}) GetInputFile(${file}) diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml index e7af45178aa5..0819562d9590 100644 --- a/components/eamxx/tests/single-process/mam/emissions/input.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -23,6 +23,8 @@ atmosphere_processes: srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc + soil_erodibility_file: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/dst_ne2np4_c20241028.nc + marine_organics_file: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/monthly_macromolecules_0.1deg_bilinear_year01_merge_ne2np4_c20241030.nc grids_manager: Type: Mesh Free geo_data_source: IC_FILE @@ -38,11 +40,15 @@ initial_conditions: Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} phis : 1.0 - #These should come from the input file - - #we should get the following variables from other processes + + # we should get the following variables from other processes pbl_height : 1.0 - + dstflx : 0.0 + ocnfrac: 0.10000000000000000E+001 + sst: 0.30178553874977507E+003 + cldfrac_tot: 0.138584624960092 + horiz_winds: [-0.24988988196194634E+000, -0.23959782871450760E+000] + constituent_fluxes: 0.0 # The parameters for I/O control Scorpio: output_yaml_files: ["output.yaml"] diff --git a/components/eamxx/tests/single-process/mam/wet_scav/output.yaml b/components/eamxx/tests/single-process/mam/wet_scav/output.yaml index 9cd0a16bc414..854d18a2e485 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/output.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/output.yaml @@ -8,7 +8,7 @@ Fields: - aerdepwetis - aerdepwetcw - dgnumwet - - dgncur_a + - dgnum - wetdens - qaerwat - bc_c1 diff --git a/components/eamxx/tests/single-process/rrtmgp/CMakeLists.txt b/components/eamxx/tests/single-process/rrtmgp/CMakeLists.txt index 792d3946a4c2..b16c2e732884 100644 --- a/components/eamxx/tests/single-process/rrtmgp/CMakeLists.txt +++ b/components/eamxx/tests/single-process/rrtmgp/CMakeLists.txt @@ -13,7 +13,7 @@ if (SCREAM_ENABLE_BASELINE_TESTS AND NOT SCREAM_ONLY_GENERATE_BASELINES) CreateUnitTest(${TEST_BASE_NAME}_unit rrtmgp_standalone_unit.cpp LABELS rrtmgp physics driver LIBS scream_rrtmgp rrtmgp scream_control yakl diagnostics rrtmgp_test_utils - EXE_ARGS "--ekat-test-params rrtmgp_inputfile=${SCREAM_DATA_DIR}/init/rrtmgp-allsky.nc,rrtmgp_baseline=${SCREAM_BASELINES_DIR}/data/rrtmgp-allsky-baseline.nc" + EXE_ARGS "--args --inputfile ${SCREAM_DATA_DIR}/init/rrtmgp-allsky.nc --baseline ${SCREAM_BASELINES_DIR}/data/rrtmgp-allsky-baseline.nc" ) endif() @@ -39,7 +39,7 @@ CreateUnitTestFromExec( ${TEST_BASE_NAME}_not_chunked ${TEST_BASE_NAME} LABELS rrtmgp physics driver MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} - EXE_ARGS "--ekat-test-params inputfile=input_not_chunked.yaml" + EXE_ARGS "--args -inputfile input_not_chunked.yaml" FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME}_not_chunked ) @@ -70,7 +70,7 @@ CreateUnitTestFromExec( ${TEST_BASE_NAME}_chunked ${TEST_BASE_NAME} LABELS rrtmgp physics driver MPI_RANKS ${TEST_RANK_END} - EXE_ARGS "--ekat-test-params inputfile=input_chunked.yaml" + EXE_ARGS "--args --inputfile=input_chunked.yaml" FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME}_chunked PROPERTIES PASS_REGULAR_EXPRESSION "(beg.end: 0, ${COL_CHUNK_SIZE})" ) diff --git a/components/eamxx/tests/single-process/rrtmgp/input_unit.yaml b/components/eamxx/tests/single-process/rrtmgp/input_unit.yaml index da1d17191aaa..4a1f474c8778 100644 --- a/components/eamxx/tests/single-process/rrtmgp/input_unit.yaml +++ b/components/eamxx/tests/single-process/rrtmgp/input_unit.yaml @@ -10,9 +10,9 @@ atmosphere_processes: active_gases: ["h2o", "co2", "o3", "n2o", "co" , "ch4", "o2", "n2"] orbital_year: 1990 # Set orbital parameters to constants for consistency with RRTMGP test problem - orbital_eccentricity: 0 - orbital_obliquity: 0 - orbital_mvelp: 0 + orbital_eccentricity: 0.0 + orbital_obliquity: 0.0 + orbital_mvelp: 0.0 Fixed Solar Zenith Angle: 0.86 do_aerosol_rad: false nswgpts: 224 diff --git a/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp b/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp index dc1ece59b287..5be27ce2b1e2 100644 --- a/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp +++ b/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp @@ -49,8 +49,8 @@ using PC = scream::physics::Constants; #ifdef RRTMGP_ENABLE_YAKL TEST_CASE("rrtmgp_scream_standalone", "") { // Get baseline name (needs to be passed as an arg) - std::string inputfile = ekat::TestSession::get().params.at("rrtmgp_inputfile"); - std::string baseline = ekat::TestSession::get().params.at("rrtmgp_baseline"); + std::string inputfile = ekat::TestSession::get().params.at("inputfile"); + std::string baseline = ekat::TestSession::get().params.at("baseline"); // Check if files exists REQUIRE(rrtmgpTest::file_exists(inputfile.c_str())); diff --git a/components/eamxx/tests/single-process/shoc/input.yaml b/components/eamxx/tests/single-process/shoc/input.yaml index befb43d98a09..179aa09c4671 100644 --- a/components/eamxx/tests/single-process/shoc/input.yaml +++ b/components/eamxx/tests/single-process/shoc/input.yaml @@ -14,7 +14,7 @@ atmosphere_processes: number_of_subcycles: ${NUM_SUBCYCLES} compute_tendencies: [all] lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/single-process/surface_coupling/surface_coupling.cpp b/components/eamxx/tests/single-process/surface_coupling/surface_coupling.cpp index a4e2a18cbfb2..73a74056d29b 100644 --- a/components/eamxx/tests/single-process/surface_coupling/surface_coupling.cpp +++ b/components/eamxx/tests/single-process/surface_coupling/surface_coupling.cpp @@ -49,7 +49,7 @@ class OutputManager4Test : public scream::OutputManager }; std::vector create_from_file_test_data(const ekat::Comm& comm, const util::TimeStamp& t0, const int ncols ) -{ +{ // Create a grids manager on the fly ekat::ParameterList gm_params; gm_params.set("grids_names",vos_type{"Point Grid"}); @@ -100,7 +100,8 @@ std::vector create_from_file_test_data(const ekat::Comm& comm, cons ctrl_pl.set("Frequency",1); ctrl_pl.set("save_grid_data",false); OutputManager4Test om; - om.setup(comm,om_pl,fm,gm,t0,false); + om.initialize(comm,om_pl,t0,false); + om.setup(fm,gm); // Create output data: // T=3600, well above the max timestep for the test. auto tw = t0; @@ -403,13 +404,13 @@ void test_exports(const FieldManager& fm, // Recall that two fields have been set to export to a constant value, so we load those constants from the parameter list here: using vor_type = std::vector; const auto prescribed_const_values = prescribed_constants.get("values"); - const Real Faxa_swndf_const = prescribed_const_values[0]; - const Real Faxa_swndv_const = prescribed_const_values[1]; + const Real Faxa_swndf_const = prescribed_const_values[0]; + const Real Faxa_swndv_const = prescribed_const_values[1]; // Check cpl data to scream fields for (int i=0; i::view_1d import_vec_comps_view ("import_vec_comps", num_scream_imports); - KokkosTypes::view_1d import_constant_multiple_view("import_constant_multiple_view", + KokkosTypes::view_1d import_constant_multiple_view("import_constant_multiple_view", num_scream_imports); KokkosTypes::view_1d do_import_during_init_view ("do_import_during_init_view", num_scream_imports); @@ -569,9 +571,9 @@ TEST_CASE("surface-coupling", "") { ncols, num_cpl_exports); KokkosTypes::view_1d export_cpl_indices_view ("export_vec_comps", num_scream_exports); - KokkosTypes::view_1d export_vec_comps_view ("export_vec_comps", + KokkosTypes::view_1d export_vec_comps_view ("export_vec_comps", num_scream_exports); - KokkosTypes::view_1d export_constant_multiple_view("export_constant_multiple_view", + KokkosTypes::view_1d export_constant_multiple_view("export_constant_multiple_view", num_scream_exports); KokkosTypes::view_1d do_export_during_init_view ("do_export_during_init_view", num_scream_exports); diff --git a/components/elm/bld/namelist_files/namelist_defaults.xml b/components/elm/bld/namelist_files/namelist_defaults.xml index b2d7a95a3fcd..3e3ce90223f7 100644 --- a/components/elm/bld/namelist_files/namelist_defaults.xml +++ b/components/elm/bld/namelist_files/namelist_defaults.xml @@ -134,7 +134,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). -lnd/clm2/paramdata/fates_params_api.36.0.0_12pft_c240517.nc +lnd/clm2/paramdata/fates_params_api.36.1.0_14pft_c241003.nc lnd/clm2/paramdata/CNP_parameters_c131108.nc @@ -232,6 +232,20 @@ ic_tod="0" sim_year="1850" glc_nec="0" use_crop=".false." >lnd/clm2/initdata_map ic_tod="0" sim_year="2000" glc_nec="0" use_crop=".false." >lnd/clm2/initdata_map/clmi.ICRUCLM45SP.2000-01-01.ne240np4_tx0.1v2.162cd32_simyr2000_c170910.nc + + +lnd/clm2/initdata/elmi.v3-SORRM.ne30pg2_r05_SOwISC12to30E3r3.1850-01-01-00000.c20240923.nc + + + + +lnd/clm2/surfdata_map/surfdata_ne0_CAx32.pg2_simyr2010_c220621.nc + +lnd/clm2/surfdata_map/surfdata_ne0_CAx32.pg2_rcp8.5_simyr1850_c220712.nc + +lnd/clm2/surfdata_map/surfdata_ne0_CAx32.pg2_rcp8.5_simyr2015_c220914.nc + lnd/clm2/surfdata_map/surfdata_ne0np4_northamericax4v1.pg2_simyr1850_c210112_with_TOP.nc @@ -327,7 +341,7 @@ lnd/clm2/surfdata_map/surfdata_64x128_simyr2000_c170111.nc lnd/clm2/surfdata_map/surfdata_nldas2_simyr2000_c181207.nc - + lnd/clm2/surfdata_map/surfdata_0.125x0.125_simyr2000_c190730.nc lnd/clm2/surfdata_map/surfdata_0.9x1.25_simyr2000_c180404.nc @@ -362,6 +376,8 @@ lnd/clm2/surfdata_map/surfdata_ne256np4_simyr2010_c220518.nc lnd/clm2/surfdata_map/surfdata_ne256np4_simyr2010_c220518.nc lnd/clm2/surfdata_map/surfdata_0.5x0.5_simyr2000_c200624.nc + +lnd/clm2/surfdata_map/surfdata_r025_simyr1980_c240920.nc lnd/clm2/surfdata_map/surfdata_0.125x0.125_simyr2000_c190730.nc @@ -398,7 +414,7 @@ lnd/clm2/surfdata_map/surfdata_360x720cru_simyr1850_c180216.nc lnd/clm2/surfdata_map/surfdata_48x96_simyr1850_c130927.nc - + lnd/clm2/surfdata_map/surfdata_0.125x0.125_simyr1850_c190730.nc lnd/clm2/surfdata_map/surfdata_0.9x1.25_simyr1850_c180306.nc @@ -615,6 +631,7 @@ this mask will have smb calculated over the entire global land surface lnd/clm2/griddata/glcmaskdata_0.9x1.25_60S.nc lnd/clm2/griddata/glcmaskdata_0.5x0.5_GIS_AIS.nc lnd/clm2/griddata/glcmaskdata_0.5x0.5_GIS_AIS.nc +lnd/clm2/griddata/glcmaskdata_r025_GIS_AIS.20240910.nc lnd/clm2/griddata/glcmaskdata_r0125_GIS_AIS.210407.nc lnd/clm2/griddata/glcmaskdata_ne30pg2_GIS_AIS.210407.nc @@ -625,6 +642,7 @@ this mask will have smb calculated over the entire global land surface lnd/clm2/griddata/topodata_48x96_USGS_070110.nc lnd/clm2/surfdata_map/surfdata_0.5x0.5_simyr1850_c200609.nc lnd/clm2/surfdata_map/surfdata_0.5x0.5_simyr1850_c211019.nc +lnd/clm2/surfdata_map/surfdata_0.25x0.25_simyr1850_c240125.nc lnd/clm2/surfdata_map/surfdata_0.125x0.125_simyr1850_c190730.nc lnd/clm2/surfdata_map/surfdata_ne30pg2_simyr1850_c210402.nc @@ -1688,13 +1706,6 @@ this mask will have smb calculated over the entire global land surface lnd/clm2/mappingdata/maps/2.5x3.33/map_1km-merge-10min_HYDRO1K-merge-nomask_to_2.5x3.33_nomask_aave_da_c130405.nc - - -lnd/clm2/mappingdata/maps/0.25x0.25/map_0.01x0.01_nomask_to_0.25x0.25_nomask_aave_da_c240501.nc - - - @@ -2058,76 +2069,83 @@ this mask will have smb calculated over the entire global land surface lnd/clm2/mappingdata/maps/northamericax4v1pg2/map_0.5x0.5_GSDTG2000_to_northamericax4v1pg2_nomask_aave_da_c210112.nc - + -lnd/clm2/mappingdata/maps/0.125x0.125/map_0.5x0.5_AVHRR_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_0.5x0.5_MODIS_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_0.9x1.25_GRDC_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_10x10min_IGBPmergeICESatGIS_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_10x10min_nomask_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_1km-merge-10min_HYDRO1K-merge-nomask_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_360x720cru_cruncep_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_3x3min_GLOBE-Gardner-mergeGIS_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_3x3min_GLOBE-Gardner_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_3x3min_LandScan2004_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_3x3min_MODIS_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_3x3min_USGS_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_5x5min_IGBP-GSDP_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_5x5min_ISRIC-WISE_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_5x5min_nomask_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_0.5x0.5_GSDTG2000_to_0.125x0.125_nomask_aave_da_c190725.nc - + -lnd/clm2/mappingdata/maps/0.25x0.25/map_0.5x0.5_AVHRR_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_0.5x0.5_MODIS_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_0.9x1.25_GRDC_to_0.25x0.25_nomask_aave_da_c240124.nc lnd/clm2/mappingdata/maps/0.25x0.25/map_10x10min_IGBPmergeICESatGIS_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_10x10min_nomask_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_1km-merge-10min_HYDRO1K-merge-nomask_to_0.25x0.25_nomask_aave_da_c240123.nc lnd/clm2/mappingdata/maps/0.25x0.25/map_360x720cru_cruncep_to_0.25x0.25_nomask_aave_da_c240124.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_3x3min_GLOBE-Gardner-mergeGIS_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_3x3min_GLOBE-Gardner_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_3x3min_LandScan2004_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_3x3min_MODIS_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_3x3min_USGS_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_5x5min_IGBP-GSDP_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_5x5min_ISRIC-WISE_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_360x720_cruncep_to_0.25x0.25_nomask_aave_da_c240124.nc +lnd/clm2/mappingdata/maps/0.25x0.25/map_5x5min_nomask_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_0.5x0.5_GSDTG2000_to_0.25x0.25_nomask_aave_da_c240123.nc +lnd/clm2/mappingdata/maps/0.25x0.25/map_0.01x0.01_nomask_to_0.25x0.25_nomask_aave_da_c240501.nc +lnd/clm2/mappingdata/maps/0.25x0.25/map_0.1x0.1_nomask_to_0.25x0.25_nomask_aave_da_c240308.nc + lnd/clm2/mappingdata/maps/antarcticax4v1/map_0.5x0.5_AVHRR_to_antarcticax4v1_nomask_aave_da_c210130.nc diff --git a/components/elm/bld/namelist_files/namelist_definition.xml b/components/elm/bld/namelist_files/namelist_definition.xml index f17a4d6ced00..87a6da722af1 100644 --- a/components/elm/bld/namelist_files/namelist_definition.xml +++ b/components/elm/bld/namelist_files/namelist_definition.xml @@ -139,6 +139,12 @@ If TRUE consider priority of plant to get a fraction of symbiotic N fixation and P phosphatase
+ +BGC balance check tolerance +Default 1.0e-7 hardwired + + Set date for beginning of adding temperature to atmospheric forcing @@ -308,7 +314,7 @@ Allowed values are: 5 : use gross domestic production and population datasets to simulate anthropogenic fire supression - Set FATES harvesting mode by setting fates_harvest_mode @@ -1477,7 +1483,7 @@ CLM run type. +"512x1024,360x720cru,128x256,64x128,48x96,32x64,8x16,94x192,0.23x0.31,0.9x1.25,1.9x2.5,2.5x3.33,4x5,10x15,5x5_amazon,1x1_tropicAtl,1x1_camdenNJ,1x1_vancouverCAN,1x1_mexicocityMEX,1x1_asphaltjungleNJ,1x1_brazil,1x1_urbanc_alpha,1x1_numaIA,1x1_smallvilleIA,0.1x0.1,0.5x0.5,3x3min,5x5min,10x10min,0.33x0.33,ne4np4,ne4np4.pg2,ne11np4,ne16np4,ne30np4,ne30np4.pg2,ne60np4,ne120np4,ne120np4.pg2,ne240np4,ne256np4,ne256np4.pg2,ne512np4.pg2,ne1024np4,ne1024np4.pg2,1km-merge-10min,ne0np4_CAx32v1.pg2,ne0np4_arm_x8v3_lowcon,ne0np4_conus_x4v1_lowcon,ne0np4_enax4v1,ne0np4_twpx4v1,r2,r05,r0125,NLDAS,ne0np4_northamericax4v1.pg2,ne0np4_arcticx4v1.pg2,r025"> Horizontal resolutions Note: 0.1x0.1, 0.5x0.5, 5x5min, 10x10min, 3x3min and 0.33x0.33 are only used for CLM tools @@ -1491,7 +1497,7 @@ Representative concentration pathway for future scenarios [radiative forcing at + valid_values="USGS,gx3v7,gx1v6,navy,test,tx0.1v2,tx1v1,T62,TL319,cruncep,oEC60to30v3,oEC60to30v3wLI,ECwISC30to60E1r2,EC30to60E2r2,WC14to60E2r3,WCAtl12to45E2r4,SOwISC12to60E2r4,ECwISC30to60E2r1,oRRS18to6,oRRS18to6v3,oRRS15to5,oARRM60to10,oARRM60to6,ARRM10to60E2r1,oQU480,oQU240,oQU240wLI,oQU120,oRRS30to10v3,oRRS30to10v3wLI,360x720cru,NLDASww3a,NLDAS,tx0.1v2,ICOS10,IcoswISC30E3r5,IcosXISC30E3r7,RRSwISC6to18E3r5,SOwISC12to30E3r3"> Land mask description @@ -1502,7 +1508,7 @@ If TRUE, irrigation will be active (find surface datasets with active irrigation +"1000,850,1100,1350,1600,1850,1855,1865,1875,1885,1895,1905,1915,1925,1935,1945,1950,1955,1965,1975,1980,1985,1995,2000,2005,2010,2015,2025,2035,2045,2055,2065,2075,2085,2095,2100,2105"> Year to simulate and to provide datasets for (such as surface datasets, initial conditions, aerosol-deposition, Nitrogen deposition rates etc.) A sim_year of 1000 corresponds to data used for testing only, NOT corresponding to any real datasets. A sim_year greater than 2005 corresponds to rcp scenario data diff --git a/components/elm/cime_config/config_compsets.xml b/components/elm/cime_config/config_compsets.xml index 5eed24e49da6..4afbdfbf3a3a 100644 --- a/components/elm/cime_config/config_compsets.xml +++ b/components/elm/cime_config/config_compsets.xml @@ -972,6 +972,11 @@ 2000_DATM%QIA_ELM%SP_SICE_SOCN_MOSART_MALI_SWAV + + IGERA5ELM_MLI + 2000_DATM%ERA5_ELM%SP_SICE_SOCN_MOSART_MALI_SWAV + + I1PTELM 2000_DATM%1PT_ELM%SP_SICE_SOCN_MOSART_SGLC_SWAV diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/lnd_rof_2way/shell_commands b/components/elm/cime_config/testdefs/testmods_dirs/elm/lnd_rof_2way/shell_commands new file mode 100644 index 000000000000..6eeec0102e5f --- /dev/null +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/lnd_rof_2way/shell_commands @@ -0,0 +1,2 @@ +./xmlchange LND_NCPL=48 +./xmlchange ROF_NCPL=48 \ No newline at end of file diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/lnd_rof_2way/user_nl_mosart b/components/elm/cime_config/testdefs/testmods_dirs/elm/lnd_rof_2way/user_nl_mosart index c7a09f76ae42..fe460362de68 100644 --- a/components/elm/cime_config/testdefs/testmods_dirs/elm/lnd_rof_2way/user_nl_mosart +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/lnd_rof_2way/user_nl_mosart @@ -1 +1,2 @@ -inundflag = .true. \ No newline at end of file +inundflag = .true. +delt_mosart = 1800 \ No newline at end of file diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/snowveg_arctic/shell_commands b/components/elm/cime_config/testdefs/testmods_dirs/elm/snowveg_arctic/shell_commands new file mode 100644 index 000000000000..396d8cea8d66 --- /dev/null +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/snowveg_arctic/shell_commands @@ -0,0 +1,10 @@ +./xmlchange LND_DOMAIN_FILE=domain_42_FLUXNETSITES_simyr1850_c170912.nc +./xmlchange ATM_DOMAIN_FILE=domain_42_FLUXNETSITES_simyr1850_c170912.nc +./xmlchange LND_DOMAIN_PATH="\$DIN_LOC_ROOT/share/domains/domain.clm" +./xmlchange ATM_DOMAIN_PATH="\$DIN_LOC_ROOT/share/domains/domain.clm" +./xmlchange DATM_MODE=CLM1PT +./xmlchange DATM_CLMNCEP_YR_START=2000 +./xmlchange DATM_CLMNCEP_YR_END=2000 +./xmlchange ELM_USRDAT_NAME=42_FLUXNETSITES +./xmlchange NTASKS=1 +./xmlchange NTHRDS=1 diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/snowveg_arctic/user_nl_elm b/components/elm/cime_config/testdefs/testmods_dirs/elm/snowveg_arctic/user_nl_elm new file mode 100644 index 000000000000..bd07678a1345 --- /dev/null +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/snowveg_arctic/user_nl_elm @@ -0,0 +1,8 @@ +fsurdat = '$DIN_LOC_ROOT/lnd/clm2/surfdata_map/surfdata_42_FLUXNETSITES_simyr1850_c170912.nc' +! this test has a parameter file with four new parameters defined for NGEE Arctic IM3 +! meant to improve snow-vegetation interactions. +! bendresist -> varies between 0 and 1, represents how much vegetation bends under snow loading +! vegshape -> suggested value 1 (parabolic), but 2 also used previously (Liston and Heimstra, 2011) +! taper -> deadstem height/radius ratio, moved from hardcoded in VegStructUpdateMod to pftvarcon +! stocking -> individual density on landscape (plants/m2), moved from hardcoded in VegStructUpdateMod to pftvarcon +paramfile = '$DIN_LOC_ROOT/lnd/clm2/paramdata/clm_params_ngeea-im3_c240822.nc' diff --git a/components/elm/docs/user-guide/index.md b/components/elm/docs/user-guide/index.md index 7995e54acb36..43fe53d66552 100644 --- a/components/elm/docs/user-guide/index.md +++ b/components/elm/docs/user-guide/index.md @@ -76,3 +76,11 @@ Using the above-mentioned settings: [FATES](fates.md) can be run in various modes with ELM through the use of namelist settings. The [FATES User's Guide section on namelist options](https://fates-users-guide.readthedocs.io/en/latest/user/Namelist-Options-and-Run-Time-Modes.html) provides guidance on enabling these different FATES run modes. + +## Create land surface dataset + +A new surface dataset for ELM is generated using `mksurfdata_map` and the notes about it are available [here](surface_dataset.md) + +## Generate land initial condition + +Initial ELM condition can be generated using `interpinic` and the notes about it are available [here](interpinic.md). diff --git a/components/elm/docs/user-guide/interpinic.md b/components/elm/docs/user-guide/interpinic.md new file mode 100644 index 000000000000..f868ee7e8d2a --- /dev/null +++ b/components/elm/docs/user-guide/interpinic.md @@ -0,0 +1,79 @@ +# Creating an ELM initial condition file + +An ELM initial condition (IC) file can be created by remapping an existing IC file from +one resolution to another using the `interpinic`, located at +`components/elm/tools/interpinic`. An ELM IC file is in the same format as an ELM restart file. +The composet of the remapped IC file will be the same as that of the input IC file. +So, for a new ELM SP-mode IC file, use an ELM input file corresponding to the SP-mode. + +The steps involved in creating a new IC files are as follows: + +1. Identifying an input ELM IC or restart file that will be remapped. +2. Obtaining an ELM restart file at the new resolution. +3. Compiling `interpinic` on the machine of interest. +4. Running `interpinic` to perform the interpolation. + +The notes below provide an example of creating 1850 ELM IC file for the NARRM grid using E3SM v3 LR piControl from year = 0101. These notes are provided for Chrysalis. + +## 1. Identification of the input ELM IC file + +The identified input land condition file for this case is the following: + +```bash +/lcrc/group/e3sm2/ac.golaz/E3SMv3/v3.LR.piControl/archive/rest/0101-01-01-00000/v3.LR.piControl.elm.r.0101-01-01-00000.nc +``` + +## 2. Obtaining an ELM restart file + +Using an existing NARRM land IC and making a copy of it + +```bash +cd components/elm/tools/interpinic + +cp /lcrc/group/e3sm/data/inputdata/lnd/clm2/initdata_map/elmi.v3-NARRM.northamericax4v1pg2_r025_IcoswISC30E3r5.1870-01-01-00000.c20240704.nc \ +elmi.v3-NARRM.northamericax4v1pg2_r025_IcoswISC30E3r5.1850-01-01-00000.c`date "+%Y%m%d"`.nc +``` + +## 3. Compiling `interpinic` + +```bash +# Load relevant modules +cd +eval $(./cime/CIME/Tools/get_case_env) + +# change directory +cd components/elm/tools/interpinic/src + +export USER_LDFLAGS="-L$NETCDF_C_DIR/lib -lnetcdf -L$NETCDF_F_DIR/lib -lnetcdff -L$HDF5_DIR/lib -lhdf5" + +USER_FC=ifort LIB_NETCDF="`nc-config --flibs`" INC_NETCDF="`nf-config --includedir`" make VERBOSE=1 + +cd ../ +``` + +## 4. Run `interpinic` + +The `interpinic` can then be run via the following batch job (e.g., `remap.r025_RRSwISC6to18E3r4.1850.batch`) to generate the initial condition. + +```bash +>cat remap.r025_RRSwISC6to18E3r4.1850.batch + +#!/bin/sh +#SBATCH --job-name=remap +#SBATCH --nodes=1 +#SBATCH --exclusive +#SBATCH --time 24:00:00 +#SBATCH -p slurm +#SBATCH --account esmd + +# Load relevant modules. +cd +eval $(./cime/CIME/Tools/get_case_env) + +# Change dir to `interpinic` +cd components/elm/tools/interpinic/ + +srun -n 1 ./interpinic \ +-i /lcrc/group/e3sm2/ac.golaz/E3SMv3/v3.LR.piControl/archive/rest/0101-01-01-00000/v3.LR.piControl.elm.r.0101-01-01-00000.nc \ +-o elmi.v3-NARRM.northamericax4v1pg2_r025_IcoswISC30E3r5.1850-01-01-00000.c20240903.nc +``` diff --git a/components/elm/docs/user-guide/surface_dataset.md b/components/elm/docs/user-guide/surface_dataset.md new file mode 100644 index 000000000000..5adaad100d6c --- /dev/null +++ b/components/elm/docs/user-guide/surface_dataset.md @@ -0,0 +1,56 @@ +# Creating an ELM surface dataset + +The notes describe the steps in creating an ELM surface dataset at 0.5x0.5 resolution for 1950 on Perlmutter. + +## 1. Load the appropriate modules + +```bash +cd +eval $(./cime/CIME/Tools/get_case_env) +``` + +## 2. Compile `mksurfdata_map` + +```bash +cd components/elm/tools/mksurfdata_map/src/ + +make clean +export USER_LDFLAGS="-L$NETCDF_DIR/lib -lnetcdf -lnetcdff -lnetcdf_intel" +export USER_LDFLAGS=$USER_LDFLAGS" -L$HDF5_DIR/lib -lhdf5 -lhdf5_fortran -lhdf5_hl_intel -lhdf5hl_fortran_intel" + +USER_FC=ifort LIB_NETCDF="`nc-config --flibs`" INC_NETCDF="`nf-config --includedir`" make VERBOSE=1 +``` + +## Build the namelist + +This step assumes that the resolution for which the new surface dataset is being created is a supported resolution. +If the surface dataset is being created for an unsupported resolution, 16 mapping files will have to be created to map the raw datasets +onto this unsupported resolution. The `namelist` file with default number of glaciers (equal to zero) can be generated as: + +```bash +cd ../ + +RES=0.5x0.5 +YR=1950 +DIN_LOC_ROOT=/global/cfs/cdirs/e3sm/inputdata + +./mksurfdata.pl -res $RES -years $YR -d -dinlc $DIN_LOC_ROOT +``` + +An example of generating the namelist for 0.25 deg (`r025`) resolution for 1980 with 10 glacier layers is as follows: + +```bash +RES=r025 +YR=1980 +DIN_LOC_ROOT=/global/cfs/cdirs/e3sm/inputdata + +./mksurfdata.pl -res $RES -years $YR -d -dinlc $DIN_LOC_ROOT -glc_nec 10 +``` + +## Run `mksurfdata_map` via an interactive job + +```bash +salloc --nodes 1 --qos interactive --time 01:00:00 --constraint cpu --account e3sm + +srun -n 1 ./mksurfdata_map < namelist +``` diff --git a/components/elm/src/biogeochem/AllocationMod.F90 b/components/elm/src/biogeochem/AllocationMod.F90 index 141194397fb0..ba14dbc352db 100644 --- a/components/elm/src/biogeochem/AllocationMod.F90 +++ b/components/elm/src/biogeochem/AllocationMod.F90 @@ -36,11 +36,11 @@ module AllocationMod use elm_varctl , only : NFIX_PTASE_plant use ELMFatesInterfaceMod , only : hlm_fates_interface_type use elm_varctl , only: iulog - use elm_varctl , only : carbon_only - use elm_varctl , only : carbonnitrogen_only + use elm_varctl , only : carbon_only + use elm_varctl , only : carbonnitrogen_only use elm_varctl , only : carbonphosphorus_only use shr_infnan_mod , only: nan => shr_infnan_nan, assignment(=) - + ! implicit none save @@ -114,23 +114,23 @@ module AllocationMod ! to toggle and update which processes are active. ! This will get set to false ! after ad_carbon_only is complete. - - + + logical :: crop_supln = .false. !Prognostic crop receives supplemental Nitrogen - + real(r8), allocatable,target :: veg_rootc_bigleaf(:,:) ! column-level fine-root biomas kgc/m3 integer, pointer :: ft_index_bigleaf(:) ! array holding the pft index of each competitor ! ECA parameters - ! scaling factor for plant fine root biomass to calculate nutrient carrier enzyme abundance - real(r8), parameter :: e_plant_scalar = 0.0000125_r8 - - ! scaling factor for plant fine root biomass to calculate nutrient carrier enzyme abundance - real(r8), parameter :: e_decomp_scalar = 0.05_r8 + ! scaling factor for plant fine root biomass to calculate nutrient carrier enzyme abundance + real(r8), parameter :: e_plant_scalar = 0.0000125_r8 + + ! scaling factor for plant fine root biomass to calculate nutrient carrier enzyme abundance + real(r8), parameter :: e_decomp_scalar = 0.05_r8 !$acc declare create(e_decomp_scalar) !$acc declare create(e_plant_scalar) - + !$acc declare copyin(crop_supln) !----------------------------------------------------------------------- @@ -213,8 +213,8 @@ subroutine AllocationInit ( bounds, elm_fates) use elm_time_manager, only: get_step_size use elm_varpar , only: crop_prog use elm_varctl , only: iulog - use elm_varctl , only : carbon_only - use elm_varctl , only : carbonnitrogen_only + use elm_varctl , only : carbon_only + use elm_varctl , only : carbonnitrogen_only use elm_varctl , only : carbonphosphorus_only @@ -234,9 +234,9 @@ subroutine AllocationInit ( bounds, elm_fates) integer :: max_comps ! maximum number of possible plant competitors ! elm big-leaf: number of pfts/patches ! fates: number of cohorts in the column - - + + !----------------------------------------------------------------------- if ( crop_prog )then @@ -253,7 +253,7 @@ subroutine AllocationInit ( bounds, elm_fates) end if end if - + ! set time steps dt = real( get_step_size(), r8 ) @@ -374,7 +374,7 @@ subroutine EvaluateSupplStatus() end subroutine EvaluateSupplStatus !------------------------------------------------------------------------------------------------- - + subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp, filter_soilp, & photosyns_vars, crop_vars, canopystate_vars, cnstate_vars, dt, yr) ! PHASE-1 of Allocation: loop over patches to assess the total plant N demand and P demand @@ -430,7 +430,7 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) froot_leaf => veg_vp%froot_leaf , & ! Input: [real(r8) (:) ] allocation parameter: new fine root C per new leaf C (gC/gC) croot_stem => veg_vp%croot_stem , & ! Input: [real(r8) (:) ] allocation parameter: new coarse root C per new stem C (gC/gC) stem_leaf => veg_vp%stem_leaf , & ! Input: [real(r8) (:) ] allocation parameter: new stem c per new leaf C (gC/gC) @@ -645,7 +645,7 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp if (stem_leaf(ivt(p)) < 0._r8) then if (stem_leaf(ivt(p)) == -1._r8) then - f3 = (2.7/(1.0+exp(-0.004*(annsum_npp(p) - 300.0)))) - 0.4 + f3 = max((2.7/(1.0+exp(-0.004*(annsum_npp(p) - 300.0)))) - 0.4_r8, 0.2_r8) else f3 = max((-1.0_r8*stem_leaf(ivt(p))*2.7_r8)/(1.0_r8+exp(-0.004_r8*(annsum_npp(p) - & 300.0_r8))) - 0.4_r8, 0.2_r8) @@ -971,7 +971,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & ! Fractional uptake profiles, that are proportional to root density real(r8):: nuptake_prof(bounds%begc:bounds%endc,1:nlevdecomp) real(r8):: puptake_prof(bounds%begc:bounds%endc,1:nlevdecomp) - integer, allocatable :: filter_pcomp(:) ! this is a plant competitor map for FATES/ELM-BL w/ ECA + integer, allocatable :: filter_pcomp(:) ! this is a plant competitor map for FATES/ELM-BL w/ ECA real(r8), allocatable,target :: plant_nh4demand_vr_fates(:,:) ! nh4 demand per competitor per soil layer real(r8), allocatable,target :: plant_no3demand_vr_fates(:,:) ! no3 demand per competitor per soil layer real(r8), allocatable,target :: plant_pdemand_vr_fates(:,:) ! p demand per competitor per soil layer @@ -990,7 +990,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & real(r8):: cp_stoich_var=0.4 ! variability of CP ratio - + !----------------------------------------------------------------------- associate( & @@ -1152,7 +1152,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & n_pcomp = elm_fates%fates(ci)%bc_out(s)%num_plant_comps pci = 1 pcf = n_pcomp - + if( nu_com.eq.'RD') then ! Overwrite the column level demands, since fates plants are all sharing @@ -1160,13 +1160,13 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & ! to scale up to column plant_ndemand_col(c) = 0._r8 plant_pdemand_col(c) = 0._r8 - + ! We fill the vertically resolved array to simplify some jointly used code do j = 1, nlevdecomp col_plant_ndemand_vr(c,j) = 0._r8 col_plant_pdemand_vr(c,j) = 0._r8 - + do f = 1,n_pcomp ft = elm_fates%fates(ci)%bc_out(s)%ft_index(f) @@ -1184,10 +1184,10 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & ! [gN/m2/s] plant_ndemand_col(c) = plant_ndemand_col(c) + col_plant_ndemand_vr(c,j)*dzsoi_decomp(j) plant_pdemand_col(c) = plant_pdemand_col(c) + col_plant_pdemand_vr(c,j)*dzsoi_decomp(j) - + end do - + else !(ECA) do f = 1,n_pcomp @@ -1195,7 +1195,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & end do veg_rootc_ptr => elm_fates%fates(ci)%bc_out(s)%veg_rootc - ft_index_ptr => elm_fates%fates(ci)%bc_out(s)%ft_index ! Should be + ft_index_ptr => elm_fates%fates(ci)%bc_out(s)%ft_index ! Should be decompmicc(:) = elm_fates%fates(ci)%bc_out(s)%decompmicc(:) ! Should be (nlevdecomp) cn_scalar_runmean_ptr => elm_fates%fates(ci)%bc_out(s)%cn_scalar ! This is 1.0 @@ -1216,7 +1216,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & pci = col_pp%pfti(c) ! Initial plant competitor index pcf = col_pp%pftf(c) ! Final plant competitor index - + if (nu_com .eq. 'RD') then do j = 1, nlevdecomp @@ -1225,7 +1225,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & end do else - + f = 0 decompmicc(:) = 0._r8 do p = col_pp%pfti(c), col_pp%pftf(c) @@ -1246,7 +1246,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & end if end do n_pcomp = f - + ft_index_ptr => ft_index_bigleaf veg_rootc_ptr => veg_rootc_bigleaf @@ -1263,7 +1263,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & (leafcn(ivt(p)) - leafcn(ivt(p))*(1- cn_stoich_var)),0.0_r8),1.0_r8) end do end if - + km_nh4_ptr => km_plant_nh4 vmax_nh4_ptr => vmax_plant_nh4 cn_scalar_runmean_ptr => cn_scalar_runmean @@ -1298,14 +1298,14 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & ! (1) add nitrogen and phosphorus competition ! (2) nitrogen and phosphorus uptake is based on root kinetics ! (3) no second pass nutrient uptake for plants - ! ============================================================= - + ! ============================================================= + if (nu_com .eq. 'RD') then ! Estimate actual allocation rates via Relative Demand ! approach (RD) - + call NAllocationRD(col_plant_ndemand_vr(c,:), & ! IN potential_immob_vr(c,:), & ! IN AllocParamsInst%compet_plant_nh4, & ! IN @@ -1332,19 +1332,19 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & ! Estimate actual allocation rates via Capacitance Aquisition ! approach (ECA/CA) - + call NAllocationECAMIC(pci,dt, & ! IN bd(c,:), & ! IN h2osoi_vol(c,:), & ! IN t_scalar(c,:), & ! IN - n_pcomp, & ! IN + n_pcomp, & ! IN filter_pcomp(1:n_pcomp), & ! IN veg_rootc_ptr(pci:pcf,:), & ! IN ft_index_ptr(pci:pcf), & ! IN cn_scalar_runmean_ptr(pci:pcf), & ! IN decompmicc, & ! IN smin_nh4_vr(c,:), & ! IN - nu_com, & ! IN + nu_com, & ! IN km_nh4_ptr, & ! IN vmax_nh4_ptr, & ! IN km_decomp_nh4, & ! IN @@ -1389,7 +1389,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & ! NO3 flux demands. supplement_to_sminn_vr(c,j) = 0._r8 if (carbon_only .or. carbonphosphorus_only) then - + if ( fpi_no3_vr(j) + fpi_nh4_vr(j) < 1._r8 ) then fpi_vr(c,j) = 1._r8 fpi_nh4_vr(j) = 1.0_r8 - fpi_no3_vr(j) @@ -1409,9 +1409,9 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & supplement_to_sminn_vr(c,j) = supplement_to_sminn_vr(c,j) + col_plant_ndemand_vr(c,j) smin_nh4_to_plant_vr(c,j) = col_plant_ndemand_vr(c,j) - smin_no3_to_plant_vr(c,j) end if - - + + end if ! sum up nitrogen limitation to decomposition @@ -1422,14 +1422,14 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & actual_immob_vr(c,j) = actual_immob_no3_vr(c,j) + actual_immob_nh4_vr(c,j) end do - + ! Starting resolving P limitation !!! ! ============================================================= if (nu_com .eq. 'RD') then ! Relative Demand (RD) - + call PAllocationRD(col_plant_pdemand_vr(c,:), & ! IN potential_immob_p_vr(c,:), & ! IN solutionp_vr(c,:), & ! IN @@ -1438,38 +1438,38 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & actual_immob_p_vr(c,:), & ! OUT sminp_to_plant_vr(c,:), & ! OUT supplement_to_sminp_vr(c,:)) ! OUT - + else call PAllocationECAMIC(pci,dt, & ! IN h2osoi_vol(c,:), & ! IN - t_scalar(c,:), & ! IN - gross_pmin_vr(c,:), & ! IN - potential_immob_p_vr(c,:), & ! IN - biochem_pmin_vr_col(c,:), & ! IN - primp_to_labilep_vr_col(c,:), & ! IN - pdep_to_sminp(c), & ! IN - pdep_prof(c,:), & ! IN - vmax_minsurf_p_vr(isoilorder(c),:), & ! IN - km_minsurf_p_vr(isoilorder(c),:), & ! IN - solutionp_vr(c,:), & ! IN - nu_com, & ! IN - n_pcomp, & ! IN + t_scalar(c,:), & ! IN + gross_pmin_vr(c,:), & ! IN + potential_immob_p_vr(c,:), & ! IN + biochem_pmin_vr_col(c,:), & ! IN + primp_to_labilep_vr_col(c,:), & ! IN + pdep_to_sminp(c), & ! IN + pdep_prof(c,:), & ! IN + vmax_minsurf_p_vr(isoilorder(c),:), & ! IN + km_minsurf_p_vr(isoilorder(c),:), & ! IN + solutionp_vr(c,:), & ! IN + nu_com, & ! IN + n_pcomp, & ! IN filter_pcomp(1:n_pcomp), & ! IN - veg_rootc_ptr(pci:pcf,:), & ! IN - ft_index_ptr(pci:pcf), & ! IN - decompmicc, & ! IN - cp_scalar_runmean_ptr(pci:pcf), & ! IN - km_p_ptr(:), & ! IN - vmax_p_ptr(:), & ! IN - km_decomp_p, & ! IN - labilep_vr(c,:), & ! IN + veg_rootc_ptr(pci:pcf,:), & ! IN + ft_index_ptr(pci:pcf), & ! IN + decompmicc, & ! IN + cp_scalar_runmean_ptr(pci:pcf), & ! IN + km_p_ptr(:), & ! IN + vmax_p_ptr(:), & ! IN + km_decomp_p, & ! IN + labilep_vr(c,:), & ! IN plant_pdemand_vr_ptr(pci:pcf,:), & ! INOUT - col_plant_pdemand_vr(c,:), & ! OUT + col_plant_pdemand_vr(c,:), & ! OUT adsorb_to_labilep_vr(c,:), & ! OUT fpi_p_vr(c,:), & ! OUT actual_immob_p_vr(c,:), & ! OUT - sminp_to_plant_vr(c,:), & ! OUT + sminp_to_plant_vr(c,:), & ! OUT desorb_to_solutionp_vr(c,:), & ! OUT supplement_to_sminp_vr(c,:)) ! OUT @@ -1482,13 +1482,13 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & supplement_to_sminp_vr(c,j) = col_plant_pdemand_vr(c,j) end do end if - + end if ! end of P competition ! resolving N limitation vs. P limitation for decomposition ! update (1) actual immobilization for N and P (2) sminn_to_plant and sminp_to_plant ! We only resolve co-limitations when are supplementing neither element - + np_bothactive: if ( .not.carbon_only .and. & .not.carbonphosphorus_only .and. & .not.carbonnitrogen_only ) then @@ -1593,7 +1593,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & actual_immob_vr(c,j) = potential_immob_vr(c,j) * fpi_p_vr(c,j) end do end if - + ! sum up plant N/P uptake at column level and patch level ! sum up N fluxes to plant after initial competition sminn_to_plant(c) = 0._r8 @@ -1602,7 +1602,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & sminn_to_plant(c) = sminn_to_plant(c) + sminn_to_plant_vr(c,j) * dzsoi_decomp(j) sminp_to_plant(c) = sminp_to_plant(c) + sminp_to_plant_vr(c,j) * dzsoi_decomp(j) end do - + ! update column plant N/P demand, pft level plant NP uptake for ECA and MIC mode eca_filter: if (nu_com .eq. 'ECA' .or. nu_com .eq. 'MIC') then @@ -1648,14 +1648,14 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & end if end do end if - + end do end if eca_filter end do col_loop - + if ((nu_com .eq. 'ECA' .or. nu_com .eq. 'MIC')) then deallocate(filter_pcomp) if(.not.use_fates)then @@ -1750,7 +1750,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & ! Set the FATES N and P uptake fluxes - + if(use_fates)then do fc=1,num_soilc c = filter_soilc(fc) @@ -1771,15 +1771,15 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & (elm_fates%fates(ci)%bc_pconst%vmax_nh4(ft)+elm_fates%fates(ci)%bc_pconst%vmax_no3(ft)) * & dzsoi_decomp(j) end do - + do j = 1,nlevdecomp - elm_fates%fates(ci)%bc_in(s)%plant_nh4_uptake_flux(f,1) = & + elm_fates%fates(ci)%bc_in(s)%plant_nh4_uptake_flux(f,1) = & elm_fates%fates(ci)%bc_in(s)%plant_nh4_uptake_flux(f,1) + & smin_nh4_to_plant_vr(c,j)*dt*dzsoi_decomp(j) * & (ndemand/plant_ndemand_col(c)) - elm_fates%fates(ci)%bc_in(s)%plant_no3_uptake_flux(f,1) = & + elm_fates%fates(ci)%bc_in(s)%plant_no3_uptake_flux(f,1) = & elm_fates%fates(ci)%bc_in(s)%plant_no3_uptake_flux(f,1) + & smin_no3_to_plant_vr(c,j)*dt*dzsoi_decomp(j) * & (ndemand/plant_ndemand_col(c)) @@ -1787,12 +1787,12 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & end do end do end if - + if( plant_pdemand_col(c)>tiny(plant_pdemand_col(c)) ) then do f = 1,n_pcomp ft = elm_fates%fates(ci)%bc_out(s)%ft_index(f) - + pdemand=0._r8 do j = 1,nlevdecomp ! [gP/m2/s] @@ -1800,14 +1800,14 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & elm_fates%fates(ci)%bc_pconst%vmax_p(ft) * & dzsoi_decomp(j) end do - + do j = 1,nlevdecomp ! [gP/m2/step] - elm_fates%fates(ci)%bc_in(s)%plant_p_uptake_flux(f,1) = & + elm_fates%fates(ci)%bc_in(s)%plant_p_uptake_flux(f,1) = & elm_fates%fates(ci)%bc_in(s)%plant_p_uptake_flux(f,1) + & sminp_to_plant_vr(c,j)*dt*dzsoi_decomp(j) * & (pdemand/plant_pdemand_col(c)) - + end do end do end if @@ -1817,21 +1817,21 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & do f = 1,n_pcomp do j = 1,nlevdecomp - elm_fates%fates(ci)%bc_in(s)%plant_nh4_uptake_flux(f,1) = & - elm_fates%fates(ci)%bc_in(s)%plant_nh4_uptake_flux(f,1) + & + elm_fates%fates(ci)%bc_in(s)%plant_nh4_uptake_flux(f,1) = & + elm_fates%fates(ci)%bc_in(s)%plant_nh4_uptake_flux(f,1) + & plant_nh4demand_vr_fates(f,j) * fpg_nh4_vr(c,j) * dzsoi_decomp(j) * dt - - elm_fates%fates(ci)%bc_in(s)%plant_no3_uptake_flux(f,1) = & - elm_fates%fates(ci)%bc_in(s)%plant_no3_uptake_flux(f,1) + & + + elm_fates%fates(ci)%bc_in(s)%plant_no3_uptake_flux(f,1) = & + elm_fates%fates(ci)%bc_in(s)%plant_no3_uptake_flux(f,1) + & plant_no3demand_vr_fates(f,j) * fpg_no3_vr(c,j) * dzsoi_decomp(j) * dt - - elm_fates%fates(ci)%bc_in(s)%plant_p_uptake_flux(f,1) = & - elm_fates%fates(ci)%bc_in(s)%plant_p_uptake_flux(f,1) + & + + elm_fates%fates(ci)%bc_in(s)%plant_p_uptake_flux(f,1) = & + elm_fates%fates(ci)%bc_in(s)%plant_p_uptake_flux(f,1) + & (plant_pdemand_vr_fates(f,j) * fpg_p_vr(c,j)) * dzsoi_decomp(j) * dt - + end do end do - + end if end do @@ -1840,7 +1840,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & deallocate(plant_no3demand_vr_fates) deallocate(plant_pdemand_vr_fates) end if - + end if ! if(use_fates) end associate @@ -1907,7 +1907,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) froot_leaf => veg_vp%froot_leaf , & ! Input: [real(r8) (:) ] allocation parameter: new fine root C per new leaf C (gC/gC) croot_stem => veg_vp%croot_stem , & ! Input: [real(r8) (:) ] allocation parameter: new coarse root C per new stem C (gC/gC) stem_leaf => veg_vp%stem_leaf , & ! Input: [real(r8) (:) ] allocation parameter: new stem c per new leaf C (gC/gC) @@ -2118,7 +2118,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & if (stem_leaf(ivt(p)) < 0._r8) then if (stem_leaf(ivt(p)) == -1._r8) then - f3 = (2.7/(1.0+exp(-0.004*(annsum_npp(p) - 300.0)))) - 0.4 + f3 = max((2.7/(1.0+exp(-0.004*(annsum_npp(p) - 300.0)))) - 0.4_r8, 0.2_r8) else f3 = max((-1.0_r8*stem_leaf(ivt(p))*2.7_r8)/(1.0_r8+exp(-0.004_r8*(annsum_npp(p) - & 300.0_r8))) - 0.4_r8, 0.2_r8) @@ -2959,7 +2959,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & end associate end subroutine Allocation3_PlantCNPAlloc - + ! ====================================================================================== subroutine NAllocationECAMIC(pci,dt, & ! IN @@ -2976,14 +2976,14 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN nu_com, & ! IN km_nh4_plant, & ! IN (pft) vmax_nh4_plant, & ! IN (pft) - km_decomp_nh4, & ! IN + km_decomp_nh4, & ! IN potential_immob_vr, & ! IN (j) plant_nh4demand_vr, & ! INOUT (i,j) col_plant_nh4demand_vr, & ! OUT (j) fpi_nh4_vr, & ! OUT (j) actual_immob_nh4_vr, & ! OUT (j) smin_nh4_to_plant_vr, & ! OUT (j) - smin_no3_vr, & ! IN (j) + smin_no3_vr, & ! IN (j) km_no3_plant, & ! IN (pft) vmax_no3_plant, & ! IN (pft) km_decomp_no3, & ! IN (j) @@ -3004,12 +3004,12 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN ! kinetics following Zhu et al., 2016 DOI: 10.1002/2016JG003554 ! ------------------------------------------------------------------------------------ use elm_varpar , only: nlevdecomp - + integer, intent(in) :: pci ! First index of plant comp arrays real(r8), intent(in) :: dt ! Time step duration [s] real(r8), intent(in) :: bd(:) ! Bulk density of dry soil material [kg m-3] real(r8), intent(in) :: h2osoi_vol(:) ! Vol. Soil Water in each layer [m3] - real(r8), intent(in) :: t_scalar(:) ! fraction by which decomposition is limited by temperature + real(r8), intent(in) :: t_scalar(:) ! fraction by which decomposition is limited by temperature integer, intent(in) :: n_pcomp ! number of plant competitors integer, intent(in) :: filter_pcomp(:) ! plant competition filter real(r8), intent(in) :: veg_rootc(pci:,:) ! total fine-root biomass of each competitor [gC/m3] @@ -3065,7 +3065,7 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN integer :: i,ip ! loop index for competitors integer :: ft ! loop index for pfts - ! 2.76 consider soil adsorption effect on [NH4+] availability, + ! 2.76 consider soil adsorption effect on [NH4+] availability, ! based on Zhu et al., 2016 DOI: 10.1002/2016JG003554 real(r8), parameter :: adsorp_nh4_eff = 2.76_r8 @@ -3073,14 +3073,14 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN do j = 1, nlevdecomp - ! Plant, microbial decomposers compete for NH4. Thus loop over each + ! Plant, microbial decomposers compete for NH4. Thus loop over each ! plant competitor in this competitive space (column). - ! Calculate competition coefficients for N/P, first need to convert - ! concentration to per soil water based + ! Calculate competition coefficients for N/P, first need to convert + ! concentration to per soil water based ! concentration of mineralized nutrient, per soil water solution_conc = smin_nh4_vr(j) / (bd(j)*adsorp_nh4_eff*m3_per_liter + h2osoi_vol(j)) - + e_km = 0._r8 do i = 1, n_pcomp ip = filter_pcomp(i) @@ -3089,14 +3089,14 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN end do e_km = e_km + e_decomp_scalar*decompmicc(j)*(1._r8/km_decomp_nh4 + 1._r8/km_nit) - + do i = 1, n_pcomp ip = filter_pcomp(i) ft = ft_index(ip) - compet_plant(i) = solution_conc / & + compet_plant(i) = solution_conc / & ( km_nh4_plant(ft) * (1._r8 + solution_conc/km_nh4_plant(ft) + e_km)) end do - + compet_decomp = solution_conc / (km_decomp_nh4 * (1._r8 + solution_conc/km_decomp_nh4 + e_km)) compet_nit = solution_conc / (km_nit * (1._r8 + solution_conc/km_nit + e_km)) @@ -3110,7 +3110,7 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN ip = filter_pcomp(i) ft = ft_index(ip) - ! This is the demand per m3 of the column (not patch) + ! This is the demand per m3 of the column (not patch) ! (for native ELM divide through by the patch weight to get per m3 of patch) plant_nh4demand_vr(ip,j) = max(0._r8,vmax_nh4_plant(ft) * veg_rootc(ip,j) * & cn_scalar_runmean(ip) * t_scalar(j) * compet_plant(i)) @@ -3130,7 +3130,7 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN else ! 'MIC' mode - sum_nh4_demand_scaled = potential_immob_vr(j)*compet_decomp + & + sum_nh4_demand_scaled = potential_immob_vr(j)*compet_decomp + & pot_f_nit_vr(j)*compet_nit end if @@ -3199,10 +3199,10 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN do i = 1, n_pcomp ip = filter_pcomp(i) ft = ft_index(ip) - compet_plant(i) = solution_conc / & + compet_plant(i) = solution_conc / & ( km_no3_plant(ft) * (1._r8 + solution_conc/km_no3_plant(ft) + e_km)) end do - + compet_decomp = solution_conc / (km_decomp_no3 * (1._r8 + solution_conc/km_decomp_no3 + e_km)) compet_denit = solution_conc / (km_den * (1._r8 + solution_conc/km_den + e_km)) @@ -3215,7 +3215,7 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN ip = filter_pcomp(i) ft = ft_index(ip) - ! This is the demand per m3 of the column (not patch) + ! This is the demand per m3 of the column (not patch) ! (for native ELM divide through by the patch weight to get per m3 of patch) plant_no3demand_vr(ip,j) = max(0._r8,vmax_no3_plant(ft) * veg_rootc(ip,j) * & cn_scalar_runmean(ip) * t_scalar(j) * compet_plant(i)) @@ -3245,7 +3245,7 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN smin_no3_to_plant_vr(j) = col_plant_no3demand_vr(j) f_denit_vr(j) = pot_f_denit_vr(j) - else + else ! NO3 availability can not satisfy the sum of immobilization, denitrification, and ! plant growth demands, so these three demands compete for available @@ -3284,34 +3284,34 @@ end subroutine NAllocationECAMIC subroutine PAllocationECAMIC(pci, & dt, & - h2osoi_vol, & - t_scalar, & - gross_pmin_vr, & - potential_immob_p_vr, & - biochem_pmin_vr_col, & - primp_to_labilep_vr_col, & - pdep_to_sminp, & - pdep_prof, & + h2osoi_vol, & + t_scalar, & + gross_pmin_vr, & + potential_immob_p_vr, & + biochem_pmin_vr_col, & + primp_to_labilep_vr_col, & + pdep_to_sminp, & + pdep_prof, & vmax_minsurf_p_vr, & km_minsurf_p_vr, & solutionp_vr, & nu_com, & n_pcomp, & - filter_pcomp, & - veg_rootc, & + filter_pcomp, & + veg_rootc, & ft_index, & decompmicc, & - cp_scalar_runmean, & + cp_scalar_runmean, & km_plant_p, & vmax_plant_p, & - km_decomp_p, & + km_decomp_p, & labilep_vr, & - plant_pdemand_vr_patch, & - col_plant_pdemand_vr, & + plant_pdemand_vr_patch, & + col_plant_pdemand_vr, & adsorb_to_labilep_vr, & fpi_p_vr, & actual_immob_p_vr, & - sminp_to_plant_vr, & + sminp_to_plant_vr, & desorb_to_solutionp_vr, & supplement_to_sminp_vr) @@ -3370,7 +3370,7 @@ subroutine PAllocationECAMIC(pci, & ! plant P uptake, microbial P uptake/release ! secondary P desorption is assumed to go into solution P pool - do j = 1, nlevdecomp + do j = 1, nlevdecomp ! plant, microbial decomposer, mineral surface compete for P ! loop over each pft within the same column @@ -3390,27 +3390,27 @@ subroutine PAllocationECAMIC(pci, & do i = 1,n_pcomp ip = filter_pcomp(i) ft = ft_index(ip) - compet_plant(i) = solution_pconc / & + compet_plant(i) = solution_pconc / & (km_plant_p(ft)*(1._r8 + solution_pconc/km_plant_p(ft) + e_km_p)) end do - + compet_decomp_p = solution_pconc / & (km_decomp_p * (1._r8 + solution_pconc/km_decomp_p + e_km_p)) - compet_minsurf_p = solution_pconc/ & + compet_minsurf_p = solution_pconc/ & (km_minsurf_p_vr(j) * (1._r8 + solution_pconc/km_minsurf_p_vr(j) + e_km_p)) col_plant_pdemand_vr(j) = 0._r8 do i = 1,n_pcomp ip = filter_pcomp(i) ft = ft_index(ip) - plant_pdemand_vr_patch(ip,j) = max(0._r8,vmax_plant_p(ft) * veg_rootc(ip,j) * & + plant_pdemand_vr_patch(ip,j) = max(0._r8,vmax_plant_p(ft) * veg_rootc(ip,j) * & cp_scalar_runmean(ip) * t_scalar(j) * compet_plant(i)) col_plant_pdemand_vr(j) = col_plant_pdemand_vr(j) + plant_pdemand_vr_patch(ip,j) end do ! potential adsorption rate without plant and microbial interaction - ! including weathering, deposition, phosphatase, mineralization, + ! including weathering, deposition, phosphatase, mineralization, ! immobilization, plant uptake dsolutionp_dt = gross_pmin_vr(j) -potential_immob_p_vr(j) - & col_plant_pdemand_vr(j) + biochem_pmin_vr_col(j) + & @@ -3477,7 +3477,7 @@ subroutine PAllocationECAMIC(pci, & if (nu_com .eq. 'MIC') sminp_to_plant_vr(j) = min(max( 0._r8, & (solutionp_vr(j)/dt) - actual_immob_p_vr(j) - adsorb_to_labilep_vr(j) ), & - col_plant_pdemand_vr(j)) + col_plant_pdemand_vr(j)) end if end do @@ -3488,7 +3488,7 @@ end subroutine PAllocationECAMIC subroutine NAllocationRD(col_plant_ndemand_vr, &! IN (j) potential_immob_vr, & ! IN (j) - compet_plants_nh4, & ! IN + compet_plants_nh4, & ! IN compet_decomp_nh4, & ! IN dt, & ! IN smin_nh4_vr, & ! IN (j) @@ -3641,7 +3641,7 @@ end subroutine NAllocationRD ! ====================================================================================== - subroutine PAllocationRD(col_plant_pdemand_vr, & ! IN + subroutine PAllocationRD(col_plant_pdemand_vr, & ! IN potential_immob_p_vr, & ! IN (j) solutionp_vr, & ! IN (j) dt, & ! IN @@ -3684,7 +3684,7 @@ subroutine PAllocationRD(col_plant_pdemand_vr, & ! IN actual_immob_p_vr(j) = potential_immob_p_vr(j) sminp_to_plant_vr(j) = col_plant_pdemand_vr(j) supplement_to_sminp_vr(j) = sum_pdemand - (solutionp_vr(j)/dt) - + else ! P availability can not satisfy the sum of immobilization and ! plant growth demands, so these two demands compete for @@ -3702,7 +3702,7 @@ subroutine PAllocationRD(col_plant_pdemand_vr, & ! IN fpi_p_vr(j) = 0.0_r8 end if - sminp_to_plant_vr(j) = max( 0._r8,(solutionp_vr(j)/dt) - actual_immob_p_vr(j) ) + sminp_to_plant_vr(j) = max( 0._r8,(solutionp_vr(j)/dt) - actual_immob_p_vr(j) ) end if end do @@ -3749,7 +3749,7 @@ subroutine calc_nuptake_prof(bounds, num_soilc, filter_soilc, cnstate_vars, nupt c = filter_soilc(fc) sminn_vr_loc(c,j) = smin_no3_vr(c,j) + smin_nh4_vr(c,j) - + if(use_pflotran .and. pf_cmode) then sminn_tot(c) = sminn_tot(c) + sminn_vr_loc(c,j) * dzsoi_decomp(j) & *(nfixation_prof(c,j)*dzsoi_decomp(j)) ! weighted by froot fractions in annual max. active layers diff --git a/components/elm/src/biogeochem/CH4Mod.F90 b/components/elm/src/biogeochem/CH4Mod.F90 index 3b3c1fa80611..a5cbd2f041c2 100644 --- a/components/elm/src/biogeochem/CH4Mod.F90 +++ b/components/elm/src/biogeochem/CH4Mod.F90 @@ -223,7 +223,7 @@ subroutine InitAllocate(this, bounds) ! Allocate module variables and data structures ! ! !USES: - use shr_infnan_mod, only: spval => shr_infnan_nan, assignment(=) + use shr_infnan_mod, only: nan => shr_infnan_nan, assignment(=) use elm_varpar , only: nlevgrnd ! ! !ARGUMENTS: diff --git a/components/elm/src/biogeochem/CNAllocationBetrMod.F90 b/components/elm/src/biogeochem/CNAllocationBetrMod.F90 index 8cfc342bb404..32f9fc6eb23b 100644 --- a/components/elm/src/biogeochem/CNAllocationBetrMod.F90 +++ b/components/elm/src/biogeochem/CNAllocationBetrMod.F90 @@ -26,13 +26,13 @@ module CNAllocationBeTRMod use PhotosynthesisType , only : photosyns_type use CropType , only : crop_type use VegetationPropertiesType, only : veg_vp - use LandunitType , only : lun_pp + use LandunitType , only : lun_pp use ColumnType , only : col_pp - use ColumnDataType , only : col_cf, col_ns, col_nf, col_ps, col_pf + use ColumnDataType , only : col_cf, col_ns, col_nf, col_ps, col_pf use VegetationType , only : veg_pp use VegetationDataType , only : veg_cs, veg_ns, veg_nf, veg_ps, veg_pf - use VegetationDataType , only : veg_cf, c13_veg_cf, c14_veg_cf - + use VegetationDataType , only : veg_cf, c13_veg_cf, c14_veg_cf + ! bgc interface & pflotran module switches use elm_varctl , only : nu_com use SoilStatetype , only : soilstate_type @@ -359,7 +359,7 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) froot_leaf => veg_vp%froot_leaf , & ! Input: [real(r8) (:) ] allocation parameter: new fine root C per new leaf C (gC/gC) croot_stem => veg_vp%croot_stem , & ! Input: [real(r8) (:) ] allocation parameter: new coarse root C per new stem C (gC/gC) stem_leaf => veg_vp%stem_leaf , & ! Input: [real(r8) (:) ] allocation parameter: new stem c per new leaf C (gC/gC) @@ -1167,7 +1167,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type ! - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) froot_leaf => veg_vp%froot_leaf , & ! Input: [real(r8) (:) ] allocation parameter: new fine root C per new leaf C (gC/gC) croot_stem => veg_vp%croot_stem , & ! Input: [real(r8) (:) ] allocation parameter: new coarse root C per new stem C (gC/gC) stem_leaf => veg_vp%stem_leaf , & ! Input: [real(r8) (:) ] allocation parameter: new stem c per new leaf C (gC/gC) diff --git a/components/elm/src/biogeochem/CNGapMortalityBeTRMod.F90 b/components/elm/src/biogeochem/CNGapMortalityBeTRMod.F90 index 3bbbf1db1f60..ed8811f61f89 100644 --- a/components/elm/src/biogeochem/CNGapMortalityBeTRMod.F90 +++ b/components/elm/src/biogeochem/CNGapMortalityBeTRMod.F90 @@ -18,9 +18,9 @@ module CNGapMortalityBeTRMod use ColumnType , only : col_pp use ColumnDataType , only : col_cf, col_nf, col_pf use VegetationPropertiesType , only : veg_vp - use VegetationType , only : veg_pp - use VegetationDataType , only : veg_cs, veg_cf, veg_ns, veg_nf - use VegetationDataType , only : veg_ps, veg_pf + use VegetationType , only : veg_pp + use VegetationDataType , only : veg_cs, veg_cf, veg_ns, veg_nf + use VegetationDataType , only : veg_ps, veg_pf use PhosphorusFluxType , only : phosphorusflux_type use PhosphorusStateType , only : phosphorusstate_type @@ -121,7 +121,7 @@ subroutine CNGapMortality (& associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody & ! Input: [real(r8) (:) ] binary flag for woody lifeform + woody => veg_vp%woody & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) ) ! set the mortality rate based on annual rate diff --git a/components/elm/src/biogeochem/CNNStateUpdate1BeTRMod.F90 b/components/elm/src/biogeochem/CNNStateUpdate1BeTRMod.F90 index 465c92f82b4c..b785f52549e7 100644 --- a/components/elm/src/biogeochem/CNNStateUpdate1BeTRMod.F90 +++ b/components/elm/src/biogeochem/CNNStateUpdate1BeTRMod.F90 @@ -59,7 +59,7 @@ subroutine NStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp, & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) cascade_donor_pool => decomp_cascade_con%cascade_donor_pool , & ! Input: [integer (:) ] which pool is C taken from for a given decomposition step cascade_receiver_pool => decomp_cascade_con%cascade_receiver_pool , & ! Input: [integer (:) ] which pool is C added to for a given decomposition step diff --git a/components/elm/src/biogeochem/CNPhenologyBeTRMod.F90 b/components/elm/src/biogeochem/CNPhenologyBeTRMod.F90 index ac3b58538e8f..066dcfb62c82 100644 --- a/components/elm/src/biogeochem/CNPhenologyBeTRMod.F90 +++ b/components/elm/src/biogeochem/CNPhenologyBeTRMod.F90 @@ -525,7 +525,7 @@ subroutine CNSeasonDecidPhenology (num_soilp, filter_soilp , & prev_dayl => grc_pp%prev_dayl , & ! Input: [real(r8) (:) ] daylength from previous time step (s) season_decid => veg_vp%season_decid , & ! Input: [real(r8) (:) ] binary flag for seasonal-deciduous leaf habit (0 or 1) - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) t_soisno => col_es%t_soisno , & ! Input: [real(r8) (:,:) ] soil temperature (Kelvin) (-nlevsno+1:nlevgrnd) @@ -875,7 +875,7 @@ subroutine CNStressDecidPhenology (num_soilp, filter_soilp , & dayl => grc_pp%dayl , & ! Input: [real(r8) (:) ] daylength (s) leaf_long => veg_vp%leaf_long , & ! Input: [real(r8) (:) ] leaf longevity (yrs) - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) stress_decid => veg_vp%stress_decid , & ! Input: [real(r8) (:) ] binary flag for stress-deciduous leaf habit (0 or 1) soilpsi => soilstate_vars%soilpsi_col , & ! Input: [real(r8) (:,:) ] soil water potential in each soil layer (MPa) @@ -2295,7 +2295,7 @@ subroutine CNOnsetGrowth (num_soilp, filter_soilp, & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) onset_flag => cnstate_vars%onset_flag_patch , & ! Input: [real(r8) (:) ] onset flag onset_counter => cnstate_vars%onset_counter_patch , & ! Input: [real(r8) (:) ] onset days counter @@ -2880,7 +2880,7 @@ subroutine CNLivewoodTurnover (num_soilp, filter_soilp, & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) livewdcn => veg_vp%livewdcn , & ! Input: [real(r8) (:) ] live wood (phloem and ray parenchyma) C:N (gC/gN) deadwdcn => veg_vp%deadwdcn , & ! Input: [real(r8) (:) ] dead wood (xylem and heartwood) C:N (gC/gN) diff --git a/components/elm/src/biogeochem/CarbonStateUpdate1Mod.F90 b/components/elm/src/biogeochem/CarbonStateUpdate1Mod.F90 index 1eb7f64d025a..5133575abd30 100644 --- a/components/elm/src/biogeochem/CarbonStateUpdate1Mod.F90 +++ b/components/elm/src/biogeochem/CarbonStateUpdate1Mod.F90 @@ -200,7 +200,7 @@ subroutine CarbonStateUpdate1(bounds, & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) cascade_donor_pool => decomp_cascade_con%cascade_donor_pool , & ! Input: [integer (:) ] which pool is C taken from for a given decomposition step cascade_receiver_pool => decomp_cascade_con%cascade_receiver_pool , & ! Input: [integer (:) ] which pool is C added to for a given decomposition step harvdate => crop_vars%harvdate_patch & ! Input: [integer (:) ] harvest date diff --git a/components/elm/src/biogeochem/ComputeSeedMod.F90 b/components/elm/src/biogeochem/ComputeSeedMod.F90 index 9b7ebf8838dc..2c2f781bdb59 100644 --- a/components/elm/src/biogeochem/ComputeSeedMod.F90 +++ b/components/elm/src/biogeochem/ComputeSeedMod.F90 @@ -183,16 +183,10 @@ subroutine LeafProportions(pft_type, ignore_current_state, & pstorage = 0._r8 pxfer = 0._r8 - if (tot_leaf == 0._r8 .or. ignore_current_state) then - if (veg_vp%evergreen(pft_type) == 1._r8) then - pleaf = 1._r8 - else - pstorage = 1._r8 - end if + if (veg_vp%evergreen(pft_type) == 1._r8) then + pleaf = 1._r8 else - pleaf = leaf /tot_leaf - pstorage = leaf_storage/tot_leaf - pxfer = leaf_xfer /tot_leaf + pstorage = 1._r8 end if end subroutine LeafProportions diff --git a/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 b/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 index ffe1be5d909b..1a9b59f776ea 100644 --- a/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 +++ b/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 @@ -48,7 +48,10 @@ module EcosystemBalanceCheckMod implicit none save private - real(r8), parameter :: balance_check_tolerance = 1e-8_r8 + + ! This corersponds to namelist variable bgc_balance_check_tolerance + real(r8), public :: balance_check_tolerance = 1e-7_r8 + ! ! !PUBLIC MEMBER FUNCTIONS: public :: BeginColCBalance @@ -278,7 +281,7 @@ subroutine ColCBalanceCheck(bounds, & end if ! check for significant errors - if (abs(col_errcb(c)) > 1e-8_r8) then + if (abs(col_errcb(c)) > balance_check_tolerance) then err_found = .true. err_index = c end if @@ -498,7 +501,7 @@ subroutine ColNBalanceCheck(bounds, & ! here is '-' adjustment. It says that the adding to PF decomp n pools was less. end if - if (abs(col_errnb(c)) > 1e-8_r8) then + if (abs(col_errnb(c)) > balance_check_tolerance) then err_found = .true. err_index = c end if @@ -731,7 +734,7 @@ subroutine ColPBalanceCheck(bounds, & col_errpb(c) = (col_pinputs(c) - col_poutputs(c))*dt - & (col_endpb(c) - col_begpb(c)) - if (abs(col_errpb(c)) > 1e-8_r8) then + if (abs(col_errpb(c)) > balance_check_tolerance) then err_found = .true. err_index = c end if diff --git a/components/elm/src/biogeochem/GrowthRespMod.F90 b/components/elm/src/biogeochem/GrowthRespMod.F90 index cb7ad522265f..cc6707d723b0 100644 --- a/components/elm/src/biogeochem/GrowthRespMod.F90 +++ b/components/elm/src/biogeochem/GrowthRespMod.F90 @@ -51,7 +51,7 @@ subroutine GrowthResp(num_soilp, filter_soilp) associate( & ivt => veg_pp%itype , & ! Input: [integer (:)] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:)] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:)] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) cpool_to_leafc => veg_cf%cpool_to_leafc , & ! Input: [real(r8) (:)] cpool_to_leafc_storage => veg_cf%cpool_to_leafc_storage , & ! Input: [real(r8) (:)] @@ -98,7 +98,7 @@ subroutine GrowthResp(num_soilp, filter_soilp) ) ! Loop through patches - ! start pft loop + ! start pft loop do fp = 1,num_soilp p = filter_soilp(fp) diff --git a/components/elm/src/biogeochem/MaintenanceRespMod.F90 b/components/elm/src/biogeochem/MaintenanceRespMod.F90 index 54005275b296..829e5c742a06 100644 --- a/components/elm/src/biogeochem/MaintenanceRespMod.F90 +++ b/components/elm/src/biogeochem/MaintenanceRespMod.F90 @@ -70,7 +70,7 @@ subroutine readMaintenanceRespParams ( ncid ) call ncd_io(varname=trim(tString),data=tempr, flag='read', ncid=ncid, readvar=readv) if ( .not. readv ) call endrun(msg=trim(errCode)//trim(tString)//errMsg(__FILE__, __LINE__)) br_mr_Inst = tempr - + end subroutine readMaintenanceRespParams !----------------------------------------------------------------------- @@ -107,7 +107,7 @@ subroutine MaintenanceResp(bounds, & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] patch vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) br_xr => veg_vp%br_xr , & ! Input: [real(r8) (:) ] base rate for excess respiration frac_veg_nosno => canopystate_vars%frac_veg_nosno_patch , & ! Input: [integer (:) ] fraction of vegetation not covered by snow (0 OR 1) [-] laisun => canopystate_vars%laisun_patch , & ! Input: [real(r8) (:) ] sunlit projected leaf area index @@ -160,7 +160,7 @@ subroutine MaintenanceResp(bounds, & ! calculate temperature corrections for each soil layer, for use in ! estimating fine root maintenance respiration with depth tcsoi(c,j) = Q10**((t_soisno(c,j)-SHR_CONST_TKFRZ - 20.0_r8)/10.0_r8) - + end do end do diff --git a/components/elm/src/biogeochem/NitrogenStateUpdate1Mod.F90 b/components/elm/src/biogeochem/NitrogenStateUpdate1Mod.F90 index c1b2bfc5d65a..f95b9a258cb1 100644 --- a/components/elm/src/biogeochem/NitrogenStateUpdate1Mod.F90 +++ b/components/elm/src/biogeochem/NitrogenStateUpdate1Mod.F90 @@ -130,7 +130,7 @@ subroutine NitrogenStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) cascade_donor_pool => decomp_cascade_con%cascade_donor_pool , & ! Input: [integer (:) ] which pool is C taken from for a given decomposition step cascade_receiver_pool => decomp_cascade_con%cascade_receiver_pool , & ! Input: [integer (:) ] which pool is C added to for a given decomposition step @@ -147,7 +147,7 @@ subroutine NitrogenStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp do j = 1, nlevdecomp do fc = 1,num_soilc c = filter_soilc(fc) - + ! N deposition and fixation (put all into NH4 pool) col_ns%smin_nh4_vr(c,j) = col_ns%smin_nh4_vr(c,j) + col_nf%ndep_to_sminn(c)*dt * ndep_prof(c,j) col_ns%smin_nh4_vr(c,j) = col_ns%smin_nh4_vr(c,j) + col_nf%nfix_to_sminn(c)*dt * nfixation_prof(c,j) @@ -219,7 +219,7 @@ subroutine NitrogenStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp end do end if end do - + do j = 1, nlevdecomp ! column loop do fc = 1,num_soilc @@ -254,8 +254,8 @@ subroutine NitrogenStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp end do ! end of column loop end do - - endif !end if is_active_betr_bgc + + endif !end if is_active_betr_bgc ! forest fertilization call get_curr_date(kyr, kmo, kda, mcsec) diff --git a/components/elm/src/biogeochem/PhenologyFluxLimitMod.F90 b/components/elm/src/biogeochem/PhenologyFluxLimitMod.F90 index 87b9637d5512..d80b844e6009 100644 --- a/components/elm/src/biogeochem/PhenologyFluxLimitMod.F90 +++ b/components/elm/src/biogeochem/PhenologyFluxLimitMod.F90 @@ -397,7 +397,7 @@ subroutine InitPhenoFluxLimiter() call spm_list_insert(spm_list, -1._r8, f_gresp_storage_to_xfer , s_gresp_storage,nelms) !turn the list into sparse matrix form call spm_list_to_mat(spm_list, spm_carbon_d, nelms, f_gresp_storage_to_xfer) - + !initialize stoichiometric relationship between carbon production flux and corresponding state varaibles call spm_list_init(spm_list, 1._r8, f_cpool_to_leafc , s_leafc, nelms) call spm_list_insert(spm_list, 1._r8, f_leafc_xfer_to_leafc , s_leafc, nelms) @@ -432,7 +432,7 @@ subroutine InitPhenoFluxLimiter() call spm_list_insert(spm_list, 1._r8, f_gresp_storage_to_xfer , s_gresp_xfer,nelms) call spm_list_insert(spm_list, 1._r8, f_cpool_to_gresp_storage , s_gresp_storage, nelms) - + !turn the list into sparse matrix form call spm_list_to_mat(spm_list, spm_carbon_p, nelms, f_gresp_storage_to_xfer) !initialize stoichiometry relationship between nutrient consumption and corresponding state variables @@ -474,7 +474,7 @@ subroutine InitPhenoFluxLimiter() call spm_list_insert(spm_list, -1._r8, f_grainn_to_food , s_grainn, nelms) call spm_list_insert(spm_list, -1._r8, f_grainn_xfer_to_grainn , s_grainn_xfer,nelms) call spm_list_insert(spm_list, -1._r8, f_retransn_to_npool , s_retransn, nelms) - !turn the list into sparse matrix form + !turn the list into sparse matrix form call spm_list_to_mat(spm_list, spm_nutrient_d, nelms, f_supplement_to_plantn) !initialize stoichiometry relationship between nutrient production and corresponding state variables call spm_list_init(spm_list, 1._r8, f_retransn_to_npool , s_npool, nelms) @@ -512,7 +512,7 @@ subroutine InitPhenoFluxLimiter() call spm_list_insert(spm_list, 1._r8, f_frootn_to_retransn , s_retransn, nelms) call spm_list_insert(spm_list, 1._r8, f_livestemn_to_retransn , s_retransn, nelms) call spm_list_insert(spm_list, 1._r8, f_livecrootn_to_retransn , s_retransn, nelms) - !turn the list into sparse matrix form + !turn the list into sparse matrix form call spm_list_to_mat(spm_list, spm_nutrient_p, nelms, f_supplement_to_plantn) end subroutine InitPhenoFluxLimiter !--------------------------------------------------------------------------- @@ -613,7 +613,7 @@ subroutine carbon_flux_limiter(bounds, num_soilc, filter_soilc,& real(r8) :: ar_p associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) harvdate => crop_vars%harvdate_patch & ! Input: [integer (:) ] harvest date ) ! set time steps @@ -847,7 +847,7 @@ subroutine nitrogen_flux_limiter(bounds, num_soilc, filter_soilc,& real(r8) :: dt associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) nf => veg_nf , & ns => veg_ns & ) @@ -1031,7 +1031,7 @@ subroutine phosphorus_flux_limiter(bounds, num_soilc, filter_soilc,& associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) pf => veg_pf , & ps => veg_ps & ) diff --git a/components/elm/src/biogeochem/PhenologyMod.F90 b/components/elm/src/biogeochem/PhenologyMod.F90 index 08e9b94da524..c46e251b60cd 100644 --- a/components/elm/src/biogeochem/PhenologyMod.F90 +++ b/components/elm/src/biogeochem/PhenologyMod.F90 @@ -163,7 +163,7 @@ subroutine readPhenolParams ( ncid ) allocate(PhenolParamsInst%lwtop ) ! ! read in parameters - ! + ! tString='crit_dayl' call ncd_io(varname=trim(tString),data=tempr, flag='read', ncid=ncid, readvar=readv) if ( .not. readv ) call endrun( msg=trim(errCode)//trim(tString)//errMsg(__FILE__, __LINE__)) @@ -175,7 +175,7 @@ subroutine readPhenolParams ( ncid ) else tString='crit_dayl_stress' call ncd_io(varname=trim(tString),data=tempr, flag='read', ncid=ncid, readvar=readv) - if ( .not. readv ) then + if ( .not. readv ) then PhenolParamsInst%crit_dayl_stress = secspqtrday else PhenolParamsInst%crit_dayl_stress = tempr @@ -238,7 +238,7 @@ subroutine readPhenolParams ( ncid ) tString='lwtop_ann' call ncd_io(varname=trim(tString),data=tempr, flag='read', ncid=ncid, readvar=readv) if ( .not. readv ) call endrun( msg=trim(errCode)//trim(tString)//errMsg(__FILE__, __LINE__)) - PhenolParamsInst%lwtop=tempr + PhenolParamsInst%lwtop=tempr !!!!========== Update to device ========= !!! !$acc update device(PhenolParamsInst%crit_dayl, & @@ -601,7 +601,7 @@ subroutine CNSeasonDecidPhenology (num_soilp, filter_soilp, cnstate_vars) prev_dayl => grc_pp%prev_dayl , & ! Input: [real(r8) (:) ] daylength from previous time step (s) season_decid => veg_vp%season_decid , & ! Input: [real(r8) (:) ] binary flag for seasonal-deciduous leaf habit (0 or 1) - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) t_soisno => col_es%t_soisno , & ! Input: [real(r8) (:,:) ] soil temperature (Kelvin) (-nlevsno+1:nlevgrnd) @@ -946,7 +946,7 @@ subroutine CNStressDecidPhenology (num_soilp, filter_soilp , & leaf_long => veg_vp%leaf_long , & ! Input: [real(r8) (:) ] leaf longevity (yrs) froot_long => veg_vp%froot_long , & ! Input: [real(r8) (:) ] fine root longevity (yrs) - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) stress_decid => veg_vp%stress_decid , & ! Input: [real(r8) (:) ] binary flag for stress-deciduous leaf habit (0 or 1) soilpsi => soilstate_vars%soilpsi_col , & ! Input: [real(r8) (:,:) ] soil water potential in each soil layer (MPa) @@ -1445,7 +1445,7 @@ subroutine CropPhenology(num_pcropp, filter_pcropp, & froot_long => veg_vp%froot_long , & ! Input: [real(r8) (:) ] fine root longevity (yrs) leafcn => veg_vp%leafcn , & ! Input: [real(r8) (:) ] leaf C:N (gC/gN) - manunitro => veg_vp%manunitro , & ! Input: max manure to apply (kgN/m2) + manunitro => veg_vp%manunitro , & ! Input: max manure to apply (kgN/m2) t_ref2m_min => veg_es%t_ref2m_min , & ! Input: [real(r8) (:) ] daily minimum of average 2 m height surface air temperature (K) t10 => veg_es%t_a10 , & ! Input: [real(r8) (:) ] 10-day running mean of the 2 m temperature (K) a5tmin => veg_es%t_a5min , & ! Input: [real(r8) (:) ] 5-day running mean of min 2-m temperature @@ -1988,7 +1988,7 @@ subroutine PerennialCropPhenology(num_ppercropp, filter_ppercropp, & froot_long => veg_vp%froot_long , & ! Input: [real(r8) (:) ] fine root longevity (yrs) leafcn => veg_vp%leafcn , & ! Input: [real(r8) (:) ] leaf C:N (gC/gN) leafcp => veg_vp%leafcp , & ! Input: [real(r8) (:) ] leaf C:P (gC/gP) - manunitro => veg_vp%manunitro , & ! Input: max manure to apply (kgN/m2) + manunitro => veg_vp%manunitro , & ! Input: max manure to apply (kgN/m2) t10 => veg_es%t_a10 , & ! Input: [real(r8) (:) ] 10-day running mean of the 2 m temperature (K) a10tmin => veg_es%t_a10min , & ! Input: [real(r8) (:) ] 10-day running mean of min 2-m temperature fertnitro => crop_vars%fertnitro_patch , & ! Input: [real(r8) (:) ] max fertilizer to be applied in total (kgN/m2) @@ -2016,8 +2016,8 @@ subroutine PerennialCropPhenology(num_ppercropp, filter_ppercropp, & crop_seedc_to_leaf => veg_cf%crop_seedc_to_leaf , & ! Output: [real(r8) (:) ] (gC/m2/s) seed source to PFT-level - synthfert => veg_nf%synthfert , & ! Output: [real(r8) (:) ] (gN/m2/s) fertilizer applied each timestep - manure => veg_nf%manure , & ! Output: [real(r8) (:) ] (gN/m2/s) manure applied each timestep + synthfert => veg_nf%synthfert , & ! Output: [real(r8) (:) ] (gN/m2/s) fertilizer applied each timestep + manure => veg_nf%manure , & ! Output: [real(r8) (:) ] (gN/m2/s) manure applied each timestep fert_p => veg_pf%fert_p , & ! Output: [real(r8) (:) ] (gP/m2/s) phosphorus fertilizer applied each timestep fert_counter => veg_nf%fert_counter , & ! Output: [real(r8) (:) ] >0 fertilize; <=0 not (seconds) @@ -2550,22 +2550,22 @@ subroutine CropPlantDate (num_soilp, filter_soilp, num_pcropp, filter_pcropp, & xt(p,kmo) = xt(p,kmo) + t_ref2m(p) * fracday/ndaypm(kmo) ! monthly average temperature xp(p,kmo) = xp(p,kmo) + (forc_rain(t)+forc_snow(t))*dt ! monthly average precipitation - ! calculate the potential evapotranspiration + ! calculate the potential evapotranspiration netrad = fsa(p) + eflx_lwrad_net(p) ! moved this here because it is calculated too late call calculate_eto(t_ref2m(p), netrad, eflx_soil_grnd(p), forc_pbot(t), forc_rh(t), forc_wind(t), dt, ETout) ! monthly ETo ETo(p,kmo) = ETo(p,kmo) + ETout - + ! calculate the P:PET for each month - if ( abs(ETo(p,kmo)) > 0._r8) then + if ( abs(ETo(p,kmo)) > 0._r8) then p2ETo(p,kmo) = xp(p,kmo)/ETo(p,kmo) else ! P:PET is undefined. ! Setting to a fill value ( 'spval' ) would - ! require nested if statements due to + ! require nested if statements due to ! the weighting of previous years (i.e., p2ETo and prev_p2ETo_bar ) ! So, set to zero for simplicity. p2ETo(p,kmo) = 0._r8 - end if + end if if (nyrs_crop_active(p) == 0) then ! for the first year, use last years values prev_xt_bar(p,kmo) = xt(p,kmo) @@ -2617,7 +2617,7 @@ subroutine CNOnsetGrowth (num_soilp, filter_soilp, & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) onset_flag => cnstate_vars%onset_flag_patch , & ! Input: [real(r8) (:) ] onset flag onset_counter => cnstate_vars%onset_counter_patch , & ! Input: [real(r8) (:) ] onset days counter @@ -3277,7 +3277,7 @@ subroutine CNLivewoodTurnover (num_soilp, filter_soilp) associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) livewdcn => veg_vp%livewdcn , & ! Input: [real(r8) (:) ] live wood (phloem and ray parenchyma) C:N (gC/gN) deadwdcn => veg_vp%deadwdcn , & ! Input: [real(r8) (:) ] dead wood (xylem and heartwood) C:N (gC/gN) diff --git a/components/elm/src/biogeochem/PhosphorusStateUpdate1Mod.F90 b/components/elm/src/biogeochem/PhosphorusStateUpdate1Mod.F90 index 6e1111169221..9297ab41e1bf 100644 --- a/components/elm/src/biogeochem/PhosphorusStateUpdate1Mod.F90 +++ b/components/elm/src/biogeochem/PhosphorusStateUpdate1Mod.F90 @@ -75,7 +75,7 @@ subroutine PhosphorusStateUpdateDynPatch(bounds, num_soilc_with_inactive,& c = filter_soilc_with_inactive(fc) col_ps%prod10p(c) = col_ps%prod10p(c) + col_pf%dwt_prod10p_gain(c)*dt col_ps%prod100p(c) = col_ps%prod100p(c) + col_pf%dwt_prod100p_gain(c)*dt - col_ps%prod1p(c) = col_ps%prod1p(c) + col_pf%dwt_crop_productp_gain(c)*dt + col_ps%prod1p(c) = col_ps%prod1p(c) + col_pf%dwt_crop_productp_gain(c)*dt do j = 1,nlevdecomp @@ -125,7 +125,7 @@ subroutine PhosphorusStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soi associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) cascade_donor_pool => decomp_cascade_con%cascade_donor_pool , & ! Input: [integer (:) ] which pool is C taken from for a given decomposition step cascade_receiver_pool => decomp_cascade_con%cascade_receiver_pool , & ! Input: [integer (:) ] which pool is C added to for a given decomposition step @@ -147,18 +147,18 @@ subroutine PhosphorusStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soi do j = 1, nlevdecomp do fc = 1,num_soilc c = filter_soilc(fc) - + ! plant to litter fluxes ! phenology and dynamic landcover fluxes col_pf%decomp_ppools_sourcesink(c,j,i_met_lit) = & col_pf%phenology_p_to_litr_met_p(c,j) * dt - + col_pf%decomp_ppools_sourcesink(c,j,i_cel_lit) = & col_pf%phenology_p_to_litr_cel_p(c,j) * dt - + col_pf%decomp_ppools_sourcesink(c,j,i_lig_lit) = & col_pf%phenology_p_to_litr_lig_p(c,j) * dt - + end do end do end if diff --git a/components/elm/src/biogeochem/VegStructUpdateMod.F90 b/components/elm/src/biogeochem/VegStructUpdateMod.F90 index 5943da355a04..74e135f6f0ea 100644 --- a/components/elm/src/biogeochem/VegStructUpdateMod.F90 +++ b/components/elm/src/biogeochem/VegStructUpdateMod.F90 @@ -40,6 +40,7 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & use pftvarcon , only : noveg, woody, iscft, crop use pftvarcon , only : ncorn, ncornirrig, ztopmx, laimx use pftvarcon , only : nmiscanthus, nmiscanthusirrig, nswitchgrass, nswitchgrassirrig + use pftvarcon , only : bendresist, vegshape, stocking, taper use elm_time_manager , only : get_rad_step_size use elm_varctl , only : spinup_state, spinup_mortality_factor ! @@ -58,8 +59,6 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & ! !LOCAL VARIABLES: integer :: p,c,g ! indices integer :: fp ! lake filter indices - real(r8) :: taper ! ratio of height:radius_breast_height (tree allometry) - real(r8) :: stocking ! #stems / ha (stocking density) real(r8) :: ol ! thickness of canopy layer covered by snow (m) real(r8) :: fb ! fraction of canopy layer covered by snow real(r8) :: tlai_old ! for use in Zeng tsai formula @@ -88,6 +87,9 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & z0mr => veg_vp%z0mr , & ! Input: [real(r8) (:) ] ratio of momentum roughness length to canopy top height (-) displar => veg_vp%displar , & ! Input: [real(r8) (:) ] ratio of displacement height to canopy top height (-) dwood => veg_vp%dwood , & ! Input: [real(r8) (:) ] density of wood (gC/m^3) + bendresist => veg_vp%bendresist , & ! Input: [real(r8) (:) ] resistance to bending under snow loading Sturm et al. 2005 (-) [0,1] + vegshape => veg_vp%vegshape , & ! Input: [real(r8) (:) ] vegetation shape for calculating snow buried fraction only (-) (Liston and Heimstra, 2011) + stocking => veg_vp%stocking , & ! Input: [real(r8) (:) ] Stocking density [stems / hectare] snow_depth => col_ws%snow_depth , & ! Input: [real(r8) (:) ] snow height (m) @@ -113,13 +115,6 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & dt = real( get_rad_step_size(), r8 ) - ! constant allometric parameters - taper = 200._r8 - stocking = 1000._r8 - - ! convert from stems/ha -> stems/m^2 - stocking = stocking / 10000._r8 - ! patch loop do fp = 1,num_soilp p = filter_soilp(fp) @@ -160,24 +155,16 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & if (woody(ivt(p)) >= 1.0_r8) then - ! trees and shrubs - - ! if shrubs have a squat taper - if (woody(ivt(p)) == 2.0_r8) then - taper = 10._r8 - ! otherwise have a tall taper - else - taper = 200._r8 - end if - - ! trees and shrubs for now have a very simple allometry, with hard-wired - ! stem taper (height:radius) and hard-wired stocking density (#individuals/area) + ! trees and shrubs currently have a very simple allometry, based on + ! stem taper (height:radius) and stocking density (#individuals/area) + ! taper and stocking density can be set as input variables now to + ! change from default values set in pftvarcon.F90 if (spinup_state >= 1) then - htop(p) = ((3._r8 * deadstemc(p) * spinup_mortality_factor * taper * taper)/ & - (SHR_CONST_PI * stocking * dwood(ivt(p))))**(1._r8/3._r8) + htop(p) = ((3._r8 * deadstemc(p) * spinup_mortality_factor * taper(ivt(p)) * taper(ivt(p)))/ & + (SHR_CONST_PI * stocking(ivt(p)) * dwood(ivt(p))))**(1._r8/3._r8) else - htop(p) = ((3._r8 * deadstemc(p) * taper * taper)/ & - (SHR_CONST_PI * stocking * dwood(ivt(p))))**(1._r8/3._r8) + htop(p) = ((3._r8 * deadstemc(p) * taper(ivt(p)) * taper(ivt(p)))/ & + (SHR_CONST_PI * stocking(ivt(p)) * dwood(ivt(p))))**(1._r8/3._r8) end if ! Peter Thornton, 5/3/2004 @@ -246,10 +233,12 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & ! adjust lai and sai for burying by snow. ! snow burial fraction for short vegetation (e.g. grasses) as in - ! Wang and Zeng, 2007. - if (woody(ivt(p)) >= 1.0_r8 ) then + ! Wang and Zeng et al 2007. + ! Taller vegetation (trees and shrubs) have been updated to use formulation similar to + ! Sturm et al. 2005; Liston and Hiemstra, 2011; and Belke-Brea et al. 2020 + if ( woody(ivt(p)) >= 1.0_r8 ) then ol = min( max(snow_depth(c)-hbot(p), 0._r8), htop(p)-hbot(p)) - fb = 1._r8 - ol / max(1.e-06_r8, htop(p)-hbot(p)) + fb = 1._r8 - (ol / max(1.e-06_r8, bendresist(ivt(p)) * (htop(p)-hbot(p)))) ** vegshape(ivt(p)) else fb = 1._r8 - max(min(snow_depth(c),0.2_r8),0._r8)/0.2_r8 ! 0.2m is assumed !depth of snow required for complete burial of grasses diff --git a/components/elm/src/biogeochem/VerticalProfileMod.F90 b/components/elm/src/biogeochem/VerticalProfileMod.F90 index 3f5ced72f336..1fa3db093b46 100644 --- a/components/elm/src/biogeochem/VerticalProfileMod.F90 +++ b/components/elm/src/biogeochem/VerticalProfileMod.F90 @@ -25,9 +25,9 @@ module VerticalProfileMod logical , public :: exponential_rooting_profile = .true. logical , public :: pftspecific_rootingprofile = .true. ! how steep profile is for root C inputs (1/ e-folding depth) (1/m) - real(r8), public :: rootprof_exp = 3. + real(r8), public :: rootprof_exp = 3._r8 ! how steep profile is for surface components (1/ e_folding depth) (1/m) - real(r8), public :: surfprof_exp = 10. + real(r8), public :: surfprof_exp = 10._r8 !----------------------------------------------------------------------- contains @@ -83,7 +83,8 @@ subroutine decomp_vertprofiles(bounds, & real(r8) :: ndep_prof_sum real(r8) :: nfixation_prof_sum real(r8) :: pdep_prof_sum - real(r8) :: delta = 1.e-10 + real(r8) :: delta = 1.e-10_r8 + real(r8), parameter :: smallparameter = tiny(1._r8) character(len=32) :: subname = 'decomp_vertprofiles' !----------------------------------------------------------------------- @@ -191,7 +192,7 @@ subroutine decomp_vertprofiles(bounds, & surface_prof_tot = surface_prof_tot + surface_prof(j) * dzsoi_decomp(j) end if end do - if ( (altmax_lastyear_indx(c) > 0) .and. (rootfr_tot > 0._r8) .and. (surface_prof_tot > 0._r8) ) then + if ( (altmax_lastyear_indx(c) > 0) .and. (rootfr_tot > smallparameter) .and. (surface_prof_tot > smallparameter) ) then ! where there is not permafrost extending to the surface, integrate the profiles over the active layer ! this is equivalnet to integrating over all soil layers outside of permafrost regions do j = 1, min(max(altmax_lastyear_indx(c), 1), nlevdecomp) @@ -212,10 +213,10 @@ subroutine decomp_vertprofiles(bounds, & end do else ! if fully frozen, or no roots, put everything in the top layer - froot_prof(p,1) = 1./dzsoi_decomp(1) - croot_prof(p,1) = 1./dzsoi_decomp(1) - leaf_prof(p,1) = 1./dzsoi_decomp(1) - stem_prof(p,1) = 1./dzsoi_decomp(1) + froot_prof(p,1) = 1._r8/dzsoi_decomp(1) + croot_prof(p,1) = 1._r8/dzsoi_decomp(1) + leaf_prof(p,1) = 1._r8/dzsoi_decomp(1) + stem_prof(p,1) = 1._r8/dzsoi_decomp(1) endif end do @@ -250,19 +251,19 @@ subroutine decomp_vertprofiles(bounds, & surface_prof_tot = surface_prof_tot + surface_prof(j) * dzsoi_decomp(j) end do if(col_pp%is_fates(c))then - if ( (altmax_lastyear_indx(c) > 0) .and. (surface_prof_tot > 0._r8) ) then + if ( (altmax_lastyear_indx(c) > 0) .and. (surface_prof_tot > smallparameter) ) then do j = 1,min(alt_ind, nlevbed) nfixation_prof(c,j) = surface_prof(j)/ surface_prof_tot ndep_prof(c,j) = surface_prof(j)/ surface_prof_tot pdep_prof(c,j) = surface_prof(j)/ surface_prof_tot end do else - nfixation_prof(c,1) = 1./dzsoi_decomp(1) - ndep_prof(c,1) = 1./dzsoi_decomp(1) - pdep_prof(c,1) = 1./dzsoi_decomp(1) + nfixation_prof(c,1) = 1._r8/dzsoi_decomp(1) + ndep_prof(c,1) = 1._r8/dzsoi_decomp(1) + pdep_prof(c,1) = 1._r8/dzsoi_decomp(1) endif else - if ( (altmax_lastyear_indx(c) > 0) .and. (rootfr_tot > 0._r8) .and. (surface_prof_tot > 0._r8) ) then + if ( (altmax_lastyear_indx(c) > 0) .and. (rootfr_tot > smallparameter) .and. (surface_prof_tot > smallparameter) ) then do j = 1, min(max(altmax_lastyear_indx(c), 1), nlevdecomp) nfixation_prof(c,j) = col_cinput_rootfr(c,j) / rootfr_tot if (j <= nlevbed) then @@ -271,9 +272,9 @@ subroutine decomp_vertprofiles(bounds, & end if end do else - nfixation_prof(c,1) = 1./dzsoi_decomp(1) - ndep_prof(c,1) = 1./dzsoi_decomp(1) - pdep_prof(c,1) = 1./dzsoi_decomp(1) + nfixation_prof(c,1) = 1._r8/dzsoi_decomp(1) + ndep_prof(c,1) = 1._r8/dzsoi_decomp(1) + pdep_prof(c,1) = 1._r8/dzsoi_decomp(1) endif end if end do @@ -294,9 +295,9 @@ subroutine decomp_vertprofiles(bounds, & ! check to make sure integral of all profiles = 1. do fc = 1,num_soilc c = filter_soilc(fc) - ndep_prof_sum = 0. - nfixation_prof_sum = 0. - pdep_prof_sum = 0. + ndep_prof_sum = 0._r8 + nfixation_prof_sum = 0._r8 + pdep_prof_sum = 0._r8 do j = 1, nlevdecomp ndep_prof_sum = ndep_prof_sum + ndep_prof(c,j) * dzsoi_decomp(j) nfixation_prof_sum = nfixation_prof_sum + nfixation_prof(c,j) * dzsoi_decomp(j) @@ -324,10 +325,10 @@ subroutine decomp_vertprofiles(bounds, & do fp = 1,num_soilp p = filter_soilp(fp) - froot_prof_sum = 0. - croot_prof_sum = 0. - leaf_prof_sum = 0. - stem_prof_sum = 0. + froot_prof_sum = 0._r8 + croot_prof_sum = 0._r8 + leaf_prof_sum = 0._r8 + stem_prof_sum = 0._r8 do j = 1, nlevdecomp froot_prof_sum = froot_prof_sum + froot_prof(p,j) * dzsoi_decomp(j) croot_prof_sum = croot_prof_sum + croot_prof(p,j) * dzsoi_decomp(j) @@ -336,7 +337,19 @@ subroutine decomp_vertprofiles(bounds, & end do if ( ( abs(froot_prof_sum - 1._r8) > delta ) .or. ( abs(croot_prof_sum - 1._r8) > delta ) .or. & ( abs(stem_prof_sum - 1._r8) > delta ) .or. ( abs(leaf_prof_sum - 1._r8) > delta ) ) then + c = veg_pp%column(p) write(iulog, *) 'profile sums: ', froot_prof_sum, croot_prof_sum, leaf_prof_sum, stem_prof_sum + write(iulog, *) 'c: ',c + write(iulog, *) 'altmax_lastyear_indx: ', altmax_lastyear_indx(c) + write(iulog, *) 'cinput_rootfr: ', col_cinput_rootfr(c,:) + write(iulog, *) 'dzsoi_decomp: ', dzsoi_decomp(:) + write(iulog, *) 'surface_prof: ', surface_prof(:) + write(iulog, *) 'p, itype(p), wtcol(p): ', p, veg_pp%itype(p), veg_pp%wtcol(p) + write(iulog, *) 'cinput_rootfr(p,:): ', cinput_rootfr(p,:) + write(iulog,*) 'croot_prof(p,:): ',croot_prof(p,:) + write(iulog,*) 'froot_prof(p,:): ',froot_prof(p,:) + write(iulog,*) 'leaf_prof(p,:): ',leaf_prof(p,:) + write(iulog,*) 'stem_prof(p,:): ',stem_prof(p,:) call endrun(msg=' ERROR: sum-1 > delta'//errMsg(__FILE__, __LINE__)) endif end do diff --git a/components/elm/src/biogeophys/HydrologyDrainageMod.F90 b/components/elm/src/biogeophys/HydrologyDrainageMod.F90 index 850b7eedac52..fbd4d8e2b6f3 100755 --- a/components/elm/src/biogeophys/HydrologyDrainageMod.F90 +++ b/components/elm/src/biogeophys/HydrologyDrainageMod.F90 @@ -47,7 +47,7 @@ subroutine HydrologyDrainage(bounds, & ! ! !USES: !$acc routine seq - use landunit_varcon , only : istice, istwet, istsoil, istice_mec, istcrop + use landunit_varcon , only : istice, istwet, istsoil, istice_mec, istcrop, istice use column_varcon , only : icol_roof, icol_road_imperv, icol_road_perv, icol_sunwall, icol_shadewall use elm_varcon , only : denh2o, denice, secspday use elm_varctl , only : glc_snow_persistence_max_days, use_vichydro, use_betr @@ -120,7 +120,9 @@ subroutine HydrologyDrainage(bounds, & qflx_runoff_r => col_wf%qflx_runoff_r , & ! Output: [real(r8) (:) ] Rural total runoff (qflx_drain+qflx_surf+qflx_qrgwl) (mm H2O /s) qflx_snwcp_ice => col_wf%qflx_snwcp_ice , & ! Output: [real(r8) (:) ] excess snowfall due to snow capping (mm H2O /s) [+]` qflx_glcice => col_wf%qflx_glcice , & ! Output: [real(r8) (:) ] flux of new glacier ice (mm H2O /s) - qflx_glcice_frz => col_wf%qflx_glcice_frz & ! Output: [real(r8) (:) ] ice growth (positive definite) (mm H2O/s) + qflx_glcice_frz => col_wf%qflx_glcice_frz , & ! Output: [real(r8) (:) ] ice growth (positive definite) (mm H2O/s) + qflx_glcice_diag => col_wf%qflx_glcice_diag , & ! Output: [real(r8) (:) ] flux of new glacier ice (mm H2O/s) - diagnostic, no MECs or GLC + qflx_glcice_frz_diag => col_wf%qflx_glcice_frz_diag & ! Output: [real(r8) (:) ] ice growth (positive definite) (mm H2O/s)) - diagnostic, no MECs or GLC ) ! Determine time step and step size @@ -215,6 +217,12 @@ subroutine HydrologyDrainage(bounds, & do c = bounds%begc,bounds%endc qflx_glcice_frz(c) = 0._r8 + qflx_glcice_frz_diag(c) = 0._r8 + + if (lun_pp%itype(l)==istice .and. qflx_snwcp_ice(c) > 0.0_r8) then + qflx_glcice_frz_diag(c) = qflx_snwcp_ice(c) + qflx_glcice_diag(c) = qflx_glcice_diag(c) + qflx_glcice_frz_diag(c) + endif end do do fc = 1,num_do_smb_c c = filter_do_smb_c(fc) @@ -222,10 +230,10 @@ subroutine HydrologyDrainage(bounds, & g = col_pp%gridcell(c) ! In the following, we convert glc_snow_persistence_max_days to r8 to avoid overflow if ( (snow_persistence(c) >= (real(glc_snow_persistence_max_days, r8) * secspday)) & - .or. lun_pp%itype(l) == istice_mec) then - qflx_glcice_frz(c) = qflx_snwcp_ice(c) - qflx_glcice(c) = qflx_glcice(c) + qflx_glcice_frz(c) - if (glc_dyn_runoff_routing(g)) qflx_snwcp_ice(c) = 0._r8 + .or. lun_pp%itype(l) == istice_mec ) then + qflx_glcice_frz(c) = qflx_snwcp_ice(c) + qflx_glcice(c) = qflx_glcice(c) + qflx_glcice_frz(c) + if (glc_dyn_runoff_routing(g)) qflx_snwcp_ice(c) = 0._r8 end if end do diff --git a/components/elm/src/biogeophys/SnowHydrologyMod.F90 b/components/elm/src/biogeophys/SnowHydrologyMod.F90 index f9270289d05d..49503da8ad8c 100644 --- a/components/elm/src/biogeophys/SnowHydrologyMod.F90 +++ b/components/elm/src/biogeophys/SnowHydrologyMod.F90 @@ -670,7 +670,7 @@ subroutine SnowCompaction(bounds, num_snowc, filter_snowc, & if (bi > dm) ddz1 = ddz1*exp(-46.0e-3_r8*(bi-dm)) else ddz1_fresh = (-grav * (burden(c) + wx/2._r8)) / & - (0.007_r8 * bi**(4.75_r8 + td/40._r8)) + (0.007_r8 * min(max(bi,dm),denice)**(4.75_r8 + min(td,0._r8)/40._r8)) snw_ssa = 3.e6_r8 / (denice * snw_rds(c,j)) if (snw_ssa < 50._r8) then ddz1_fresh = ddz1_fresh * exp(-46.e-2_r8 * (50._r8 - snw_ssa)) diff --git a/components/elm/src/biogeophys/SoilTemperatureMod.F90 b/components/elm/src/biogeophys/SoilTemperatureMod.F90 index d4a1074bf4a7..451b3cbccf22 100644 --- a/components/elm/src/biogeophys/SoilTemperatureMod.F90 +++ b/components/elm/src/biogeophys/SoilTemperatureMod.F90 @@ -1316,7 +1316,7 @@ subroutine Phasechange_beta (bounds, num_nolakec, filter_nolakec, dhsdT, & use elm_varctl , only : iulog use elm_varcon , only : tfrz, hfus, grav use column_varcon , only : icol_roof, icol_sunwall, icol_shadewall, icol_road_perv - use landunit_varcon , only : istsoil, istcrop, istice_mec + use landunit_varcon , only : istsoil, istcrop, istice_mec,istice ! ! !ARGUMENTS: type(bounds_type) , intent(in) :: bounds @@ -1369,6 +1369,8 @@ subroutine Phasechange_beta (bounds, num_nolakec, filter_nolakec, dhsdT, & qflx_snofrz_col => col_wf%qflx_snofrz , & ! Output: [real(r8) (:) ] column-integrated snow freezing rate (positive definite) [kg m-2 s-1] qflx_glcice => col_wf%qflx_glcice , & ! Output: [real(r8) (:) ] flux of new glacier ice (mm H2O/s) [+ = ice grows] qflx_glcice_melt => col_wf%qflx_glcice_melt , & ! Output: [real(r8) (:) ] ice melt (positive definite) (mm H2O/s) + qflx_glcice_diag => col_wf%qflx_glcice_diag , & ! Output: [real(r8) (:) ] flux of new glacier ice (mm H2O/s) [+ = ice grows] + qflx_glcice_melt_diag => col_wf%qflx_glcice_melt_diag , & ! Output: [real(r8) (:) ] ice melt (positive definite) (mm H2O/s) qflx_snomelt => col_wf%qflx_snomelt , & ! Output: [real(r8) (:) ] snow melt (mm H2O /s) eflx_snomelt => col_ef%eflx_snomelt , & ! Output: [real(r8) (:) ] snow melt heat flux (W/m**2) @@ -1393,6 +1395,7 @@ subroutine Phasechange_beta (bounds, num_nolakec, filter_nolakec, dhsdT, & qflx_snofrz_lyr(c,-nlevsno+1:0) = 0._r8 qflx_snofrz_col(c) = 0._r8 qflx_glcice_melt(c) = 0._r8 + qflx_glcice_melt_diag(c) = 0._r8 qflx_snow_melt(c) = 0._r8 end do @@ -1643,8 +1646,8 @@ subroutine Phasechange_beta (bounds, num_nolakec, filter_nolakec, dhsdT, & ! as computed in HydrologyDrainageMod.F90. l = col_pp%landunit(c) - if (lun_pp%itype(l)==istice_mec) then + if ( lun_pp%itype(l)==istice_mec) then if (j>=1 .and. h2osoi_liq(c,j) > 0._r8) then ! ice layer with meltwater ! melting corresponds to a negative ice flux qflx_glcice_melt(c) = qflx_glcice_melt(c) + h2osoi_liq(c,j)/dtime @@ -1656,6 +1659,16 @@ subroutine Phasechange_beta (bounds, num_nolakec, filter_nolakec, dhsdT, & endif ! liquid water is present endif ! istice_mec + ! for diagnostic QICE SMB output only - + ! these are to calculate SMB even without MECs + if ( lun_pp%itype(l)==istice) then + if (j>=1 .and. h2osoi_liq(c,j) > 0._r8) then ! ice layer with meltwater + ! melting corresponds to a negative ice flux + qflx_glcice_melt_diag(c) = qflx_glcice_melt_diag(c) + h2osoi_liq(c,j)/dtime + qflx_glcice_diag(c) = qflx_glcice_diag(c) - h2osoi_liq(c,j)/dtime + endif ! liquid water is present + endif ! istice_mec + end do ! end of column-loop enddo ! end of level-loop diff --git a/components/elm/src/cpl/lnd_comp_mct.F90 b/components/elm/src/cpl/lnd_comp_mct.F90 index e533d5bcbc2b..8d3ae5e2997b 100644 --- a/components/elm/src/cpl/lnd_comp_mct.F90 +++ b/components/elm/src/cpl/lnd_comp_mct.F90 @@ -18,7 +18,6 @@ module lnd_comp_mct #ifdef HAVE_MOAB use seq_comm_mct, only: mlnid! id of moab land app - use seq_comm_mct, only: mb_land_mesh! true if land is full mesh (on the river mesh) use seq_comm_mct, only: num_moab_exports #ifdef MOABCOMP use seq_comm_mct , only: seq_comm_compare_mb_mct @@ -51,7 +50,6 @@ module lnd_comp_mct integer :: mpicom_lnd_moab ! used also for mpi-reducing the difference between moab tags and mct avs integer :: rank2 - logical :: samegrid_al ! #endif !--------------------------------------------------------------------------- @@ -314,13 +312,6 @@ subroutine lnd_init_mct( EClock, cdata_l, x2l_l, l2x_l, NLFilename ) call lnd_domain_mct( bounds, lsz, gsMap_lnd, dom_l ) #ifdef HAVE_MOAB -! find out samegrid_al or not; from infodata - samegrid_al = .true. - call seq_infodata_GetData(infodata , & - atm_gnam=atm_gnam , & - lnd_gnam=lnd_gnam ) - if (trim(atm_gnam) /= trim(lnd_gnam)) samegrid_al = .false. - mb_land_mesh = .not. samegrid_al ! global variable, saved in seq_comm call init_moab_land(bounds, LNDID) #endif call mct_aVect_init(x2l_l, rList=seq_flds_x2l_fields, lsize=lsz) @@ -547,8 +538,7 @@ subroutine lnd_run_mct(EClock, cdata_l, x2l_l, l2x_l) ! loop over all fields in seq_flds_x2l_fields call mct_list_init(temp_list ,seq_flds_x2l_fields) size_list=mct_list_nitem (temp_list) - ent_type = 0 ! entity type is vertex for land, usually (bigrid case) - if (mb_land_mesh) ent_type = 1 + ent_type = 0 ! entity type is vertex for land, always if (rank2 .eq. 0) print *, num_moab_exports, trim(seq_flds_x2l_fields), ' lnd import check' do index_list = 1, size_list call mct_list_get(mctOStr,index_list,temp_list) @@ -846,7 +836,7 @@ subroutine init_moab_land(bounds, LNDID) use spmdmod , only: masterproc use iMOAB , only: iMOAB_CreateVertices, iMOAB_WriteMesh, iMOAB_RegisterApplication, & iMOAB_DefineTagStorage, iMOAB_SetIntTagStorage, iMOAB_SetDoubleTagStorage, & - iMOAB_ResolveSharedEntities, iMOAB_CreateElements, iMOAB_UpdateMeshInfo + iMOAB_ResolveSharedEntities, iMOAB_UpdateMeshInfo type(bounds_type) , intent(in) :: bounds integer , intent(in) :: LNDID ! id of the land app @@ -893,200 +883,97 @@ subroutine init_moab_land(bounds, LNDID) vgids(n) = ldecomp%gdc2glo(bounds%begg+n-1) ! local to global ! end do gsize = ldomain%ni * ldomain%nj ! size of the total grid - ! if ldomain%nv > 3 , create mesh - - ! Case where land and river share mesh (tri-grid) - if (ldomain%nv .ge. 3 .and. .not.samegrid_al) then - ! number of vertices is nv * lsz ! - allocate(moab_vert_coords(lsz*dims*ldomain%nv)) - ! loop over ldomain - allocate(moabconn(ldomain%nv * lsz)) - do n = bounds%begg, bounds%endg - i = (n - bounds%begg) * ldomain%nv - do iv = 1, ldomain%nv - lonv = ldomain%mblonv(n, iv) * SHR_CONST_PI/180. - latv = ldomain%mblatv(n, iv) * SHR_CONST_PI/180. - - i = i + 1 ! iv-th vertex of cell n; i starts at 1 - moab_vert_coords(3*i-2)=COS(latv)*COS(lonv) - moab_vert_coords(3*i-1)=COS(latv)*SIN(lonv) - moab_vert_coords(3*i )=SIN(latv) - moabconn(i) = i - enddo - enddo - ierr = iMOAB_CreateVertices(mlnid, lsz * 3 * ldomain%nv, dims, moab_vert_coords) - if (ierr > 0 ) & - call endrun('Error: fail to create MOAB vertices in land model') - - mbtype = 2 ! triangle - if (ldomain%nv .eq. 4) mbtype = 3 ! quad - if (ldomain%nv .gt. 4) mbtype = 4 ! polygon - block_ID = 100 !some value - ierr = iMOAB_CreateElements( mlnid, lsz, mbtype, ldomain%nv, moabconn, block_ID ); - - - ! define some useful tags on cells - tagtype = 0 ! dense, integer - numco = 1 - tagname='GLOBAL_ID'//C_NULL_CHAR - ierr = iMOAB_DefineTagStorage(mlnid, tagname, tagtype, numco, tagindex ) - if (ierr > 0 ) & - call endrun('Error: fail to retrieve GLOBAL_ID tag ') - - ent_type = 1 ! element type - ierr = iMOAB_SetIntTagStorage ( mlnid, tagname, lsz , ent_type, vgids) - if (ierr > 0 ) & - call endrun('Error: fail to set GLOBAL_ID tag ') - - ! use moab_vert_coords as a data holder for a frac tag and area tag that we will create - ! on the vertices; do not allocate other data array - ! Define and Set Fraction - tagname='frac'//C_NULL_CHAR - tagtype = 1 ! dense, double - ierr = iMOAB_DefineTagStorage(mlnid, tagname, tagtype, numco, tagindex ) - if (ierr > 0 ) & - call endrun('Error: fail to create frac tag ') - - do i = 1, lsz - n = i-1 + bounds%begg - moab_vert_coords(i) = ldomain%frac(n) - enddo - ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsz , ent_type, moab_vert_coords) - if (ierr > 0 ) & - call endrun('Error: fail to set frac tag ') - - ! Define and Set area - tagname='area'//C_NULL_CHAR - ierr = iMOAB_DefineTagStorage(mlnid, tagname, tagtype, numco, tagindex ) - if (ierr > 0 ) & - call endrun('Error: fail to create area tag ') - do i = 1, lsz - n = i-1 + bounds%begg - moab_vert_coords(i) = ldomain%area(n)/(re*re) ! use the same doubles for second tag :) - enddo - - ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsz , ent_type, moab_vert_coords ) - if (ierr > 0 ) & - call endrun('Error: fail to set area tag ') - - ! Define aream - tagname='aream'//C_NULL_CHAR - ierr = iMOAB_DefineTagStorage(mlnid, tagname, tagtype, numco, tagindex ) - if (ierr > 0 ) & - call endrun('Error: fail to create aream tag ') - - deallocate(moabconn) - deallocate(vgids) - - - ! Now do the verticies - allocate(vgids(lsz*ldomain%nv)) ! - do n = 1, lsz - do i=1,ldomain%nv - vgids( (n-1)*ldomain%nv+i ) = (ldecomp%gdc2glo(bounds%begg+n-1)-1)*ldomain%nv+i ! local to global ! - end do - end do - ent_type = 0 ! vertices now - tagname = 'GLOBAL_ID'//C_NULL_CHAR - ierr = iMOAB_SetIntTagStorage ( mlnid, tagname, lsz , ent_type, vgids ) - if (ierr > 0 ) & - call endrun('Error: fail to set global ID tag on vertices in land mesh ') - ierr = iMOAB_UpdateMeshInfo( mlnid ) - if (ierr > 0 ) & - call endrun('Error: fail to update mesh info ') - - ! Case where land and atmosphere share mesh - else ! old point cloud mesh - allocate(moab_vert_coords(lsz*dims)) - do i = 1, lsz - n = i-1 + bounds%begg - lonv = ldomain%lonc(n) *SHR_CONST_PI/180. - latv = ldomain%latc(n) *SHR_CONST_PI/180. - moab_vert_coords(3*i-2)=COS(latv)*COS(lonv) - moab_vert_coords(3*i-1)=COS(latv)*SIN(lonv) - moab_vert_coords(3*i )=SIN(latv) - enddo - ierr = iMOAB_CreateVertices(mlnid, lsz*3, dims, moab_vert_coords) - if (ierr > 0 ) & - call endrun('Error: fail to create MOAB vertices in land model') - - tagtype = 0 ! dense, integer - numco = 1 - tagname='GLOBAL_ID'//C_NULL_CHAR - ierr = iMOAB_DefineTagStorage(mlnid, tagname, tagtype, numco, tagindex ) - if (ierr > 0 ) & - call endrun('Error: fail to retrieve GLOBAL_ID tag ') - - ent_type = 0 ! vertex type - ierr = iMOAB_SetIntTagStorage ( mlnid, tagname, lsz , ent_type, vgids) - if (ierr > 0 ) & - call endrun('Error: fail to set GLOBAL_ID tag ') - - ierr = iMOAB_ResolveSharedEntities( mlnid, lsz, vgids ); - if (ierr > 0 ) & - call endrun('Error: fail to resolve shared entities') - - !there are no shared entities, but we will set a special partition tag, in order to see the - ! partitions ; it will be visible with a Pseudocolor plot in VisIt - tagname='partition'//C_NULL_CHAR - ierr = iMOAB_DefineTagStorage(mlnid, tagname, tagtype, numco, tagindex ) - if (ierr > 0 ) & - call endrun('Error: fail to create new partition tag ') - - vgids = iam - ierr = iMOAB_SetIntTagStorage ( mlnid, tagname, lsz , ent_type, vgids) - if (ierr > 0 ) & - call endrun('Error: fail to set partition tag ') - - ! use moab_vert_coords as a data holder for a frac tag and area tag that we will create - ! on the vertices; do not allocate other data array - tagname='frac'//C_NULL_CHAR - tagtype = 1 ! dense, double - ierr = iMOAB_DefineTagStorage(mlnid, tagname, tagtype, numco, tagindex ) - if (ierr > 0 ) & - call endrun('Error: fail to create frac tag ') - - do i = 1, lsz - n = i-1 + bounds%begg - moab_vert_coords(i) = ldomain%frac(n) - enddo - ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsz , ent_type, moab_vert_coords) - if (ierr > 0 ) & - call endrun('Error: fail to set frac tag ') - - tagname='area'//C_NULL_CHAR - ierr = iMOAB_DefineTagStorage(mlnid, tagname, tagtype, numco, tagindex ) - if (ierr > 0 ) & - call endrun('Error: fail to create area tag ') - do i = 1, lsz - n = i-1 + bounds%begg - moab_vert_coords(i) = ldomain%area(n)/(re*re) ! use the same doubles for second tag :) - enddo - - ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsz , ent_type, moab_vert_coords ) - if (ierr > 0 ) & - call endrun('Error: fail to set area tag ') - - ! aream needed in cime_init for now. - tagname='aream'//C_NULL_CHAR - ierr = iMOAB_DefineTagStorage(mlnid, tagname, tagtype, numco, tagindex ) - if (ierr > 0 ) & - call endrun('Error: fail to create aream tag ') - ! ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsz , ent_type, moab_vert_coords ) - ! if (ierr > 0 ) & - ! call endrun('Error: fail to set aream tag ') - ierr = iMOAB_UpdateMeshInfo( mlnid ) - if (ierr > 0 ) & - call endrun('Error: fail to update mesh info ') - endif - ! add more domain fields that are missing from domain fields: lat, lon, mask, hgt - tagname = 'lat:lon:mask:hgt'//C_NULL_CHAR - tagtype = 1 ! dense, double + + allocate(moab_vert_coords(lsz*dims)) + do i = 1, lsz + n = i-1 + bounds%begg + lonv = ldomain%lonc(n) *SHR_CONST_PI/180. + latv = ldomain%latc(n) *SHR_CONST_PI/180. + moab_vert_coords(3*i-2)=COS(latv)*COS(lonv) + moab_vert_coords(3*i-1)=COS(latv)*SIN(lonv) + moab_vert_coords(3*i )=SIN(latv) + enddo + ierr = iMOAB_CreateVertices(mlnid, lsz*3, dims, moab_vert_coords) + if (ierr > 0 ) & + call endrun('Error: fail to create MOAB vertices in land model') + + tagtype = 0 ! dense, integer numco = 1 + tagname='GLOBAL_ID'//C_NULL_CHAR + ierr = iMOAB_DefineTagStorage(mlnid, tagname, tagtype, numco, tagindex ) + if (ierr > 0 ) & + call endrun('Error: fail to retrieve GLOBAL_ID tag ') + + ent_type = 0 ! vertex type + ierr = iMOAB_SetIntTagStorage ( mlnid, tagname, lsz , ent_type, vgids) + if (ierr > 0 ) & + call endrun('Error: fail to set GLOBAL_ID tag ') + + ierr = iMOAB_ResolveSharedEntities( mlnid, lsz, vgids ); + if (ierr > 0 ) & + call endrun('Error: fail to resolve shared entities') + + !there are no shared entities, but we will set a special partition tag, in order to see the + ! partitions ; it will be visible with a Pseudocolor plot in VisIt + tagname='partition'//C_NULL_CHAR + ierr = iMOAB_DefineTagStorage(mlnid, tagname, tagtype, numco, tagindex ) + if (ierr > 0 ) & + call endrun('Error: fail to create new partition tag ') + + vgids = iam + ierr = iMOAB_SetIntTagStorage ( mlnid, tagname, lsz , ent_type, vgids) + if (ierr > 0 ) & + call endrun('Error: fail to set partition tag ') + + ! use moab_vert_coords as a data holder for a frac tag and area tag that we will create + ! on the vertices; do not allocate other data array + tagname='frac'//C_NULL_CHAR + tagtype = 1 ! dense, double + ierr = iMOAB_DefineTagStorage(mlnid, tagname, tagtype, numco, tagindex ) + if (ierr > 0 ) & + call endrun('Error: fail to create frac tag ') + + do i = 1, lsz + n = i-1 + bounds%begg + moab_vert_coords(i) = ldomain%frac(n) + enddo + ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsz , ent_type, moab_vert_coords) + if (ierr > 0 ) & + call endrun('Error: fail to set frac tag ') + + tagname='area'//C_NULL_CHAR ierr = iMOAB_DefineTagStorage(mlnid, tagname, tagtype, numco, tagindex ) if (ierr > 0 ) & - call endrun('Error: fail to create lat:lon:mask:hgt tags ') + call endrun('Error: fail to create area tag ') + do i = 1, lsz + n = i-1 + bounds%begg + moab_vert_coords(i) = ldomain%area(n)/(re*re) ! use the same doubles for second tag :) + enddo + + ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsz , ent_type, moab_vert_coords ) + if (ierr > 0 ) & + call endrun('Error: fail to set area tag ') + + ! aream needed in cime_init for now. + tagname='aream'//C_NULL_CHAR + ierr = iMOAB_DefineTagStorage(mlnid, tagname, tagtype, numco, tagindex ) + if (ierr > 0 ) & + call endrun('Error: fail to create aream tag ') + ! ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsz , ent_type, moab_vert_coords ) + ! if (ierr > 0 ) & + ! call endrun('Error: fail to set aream tag ') + ierr = iMOAB_UpdateMeshInfo( mlnid ) + if (ierr > 0 ) & + call endrun('Error: fail to update mesh info ') + + ! add more domain fields that are missing from domain fields: lat, lon, mask, hgt + tagname = 'lat:lon:mask:hgt'//C_NULL_CHAR + tagtype = 1 ! dense, double + numco = 1 + ierr = iMOAB_DefineTagStorage(mlnid, tagname, tagtype, numco, tagindex ) + if (ierr > 0 ) & + call endrun('Error: fail to create lat:lon:mask:hgt tags ') ! moab_vert_coords is big enough in both case to hold enough data for us: lat, lon, mask do i = 1, lsz @@ -1098,9 +985,7 @@ subroutine init_moab_land(bounds, LNDID) tagname = 'lat:lon:mask'//C_NULL_CHAR ent_type = 0 ! point cloud usually - if (ldomain%nv .ge. 3 .and. .not.samegrid_al) then - ent_type = 1 ! cell in tri-grid case - endif + ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsz*3 , ent_type, moab_vert_coords) if (ierr > 0 ) & call endrun('Error: fail to set lat lon mask tag ') @@ -1255,11 +1140,7 @@ subroutine lnd_export_moab(EClock, bounds, lnd2atm_vars, lnd2glc_vars) end do tagname=trim(seq_flds_l2x_fields)//C_NULL_CHAR - if (samegrid_al) then - ent_type = 0 ! vertices, cells only if samegrid_al false - else - ent_type = 1 - endif + ent_type = 0 ! vertices only, from now on ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, totalmbls , ent_type, l2x_lm(1,1) ) if (ierr > 0 ) & call shr_sys_abort( sub//' Error: fail to set moab l2x '// trim(seq_flds_l2x_fields) ) @@ -1452,11 +1333,7 @@ subroutine lnd_import_moab(EClock, bounds, atm2lnd_vars, glc2lnd_vars) call endrun('Error: fail to write the moab lnd mesh before import ') #endif tagname=trim(seq_flds_x2l_fields)//C_NULL_CHAR - if (samegrid_al) then - ent_type = 0 ! vertices, cells only if samegrid_al false - else - ent_type = 1 - endif + ent_type = 0 ! vertices ierr = iMOAB_GetDoubleTagStorage ( mlnid, tagname, totalmblsimp , ent_type, x2l_lm(1,1) ) if ( ierr > 0) then call endrun('Error: fail to get seq_flds_x2l_fields for land moab instance on component') diff --git a/components/elm/src/cpl/lnd_downscale_atm_forcing.F90 b/components/elm/src/cpl/lnd_downscale_atm_forcing.F90 index 99b03a2e25ae..3b69158f49eb 100644 --- a/components/elm/src/cpl/lnd_downscale_atm_forcing.F90 +++ b/components/elm/src/cpl/lnd_downscale_atm_forcing.F90 @@ -35,10 +35,13 @@ module lnd_downscale_atm_forcing ! ! !PUBLIC MEMBER FUNCTIONS: public :: downscale_atm_forcing_to_topounit ! Calls downscaling subroutines of forcing fields from gridcell to topounit + public :: downscale_atm_forcing_to_topounit_cpl_bypass ! Calls downscaling subroutines of forcing fields from gridcell to topounit, for use within the CPL_BYPASS code ! ! !PRIVATE MEMBER FUNCTIONS: private :: downscale_atm_state_to_topounit ! Downscale atmosperic state fields from gridcell to topounit + private :: downscale_atm_state_to_topounit_cpl_bypass ! Downscale atmosperic state fields from gridcell to topounit, for cpl_bypass code private :: downscale_longwave_to_topounit ! Downscale longwave radiation field from gridcell to topounit + private :: downscale_longwave_to_topounit_cpl_bypass ! Downscale longwave radiation field from gridcell to topounit, for cpl_bypass code private :: downscale_precip_to_topounit_FNM ! Downscale precipitation field from gridcell to topounit using Froude number method (FNM) private :: downscale_precip_to_topounit_ERMM ! Downscale precipitation field from gridcell to topounit using elevation ration with maximum elevation method (ERMM) private :: build_normalization ! Compute normalization factors so that downscaled fields are conservative @@ -787,4 +790,526 @@ subroutine build_normalization(orig_field, sum_field, sum_wts, norms) end subroutine build_normalization + subroutine downscale_atm_forcing_to_topounit_cpl_bypass(g, atm2lnd_vars, lnd2atm_vars) + ! + ! !DESCRIPTION: + ! Downscale fields from gridcell to topounit + ! + ! Downscaling is done over topounits if the number of topounits > 1. + ! + ! !USES: + use elm_time_manager, only : get_nstep + use elm_varcon , only : rair, cpair, grav, lapse_glcmec + use elm_varcon , only : glcmec_rain_snow_threshold, o2_molar_const + use shr_const_mod , only : SHR_CONST_TKFRZ + use landunit_varcon , only : istice_mec + use elm_varctl , only : glcmec_downscale_rain_snow_convert + use domainMod , only : ldomain + use QsatMod , only : Qsat + use FrictionVelocityMod, only: atm_gustiness + ! + ! !ARGUMENTS: + integer , intent(in) :: g + type(atm2lnd_type) , intent(in) :: atm2lnd_vars + type(lnd2atm_type) , intent(in) :: lnd2atm_vars + + ! + ! !LOCAL VARIABLES: + integer :: t, l, c, fc, t2 ! indices + integer :: clo, cc + integer :: numt_pg ! Number of topounits per grid + + ! temporaries for topo downscaling + real(r8) :: rain_g, snow_g + real(r8) :: mxElv ! Maximum elevation value per grid + real(r8) :: uovern_t ! Froude Number + real(r8) :: grdElv ! Grid elevation + real(r8) :: topoElv ! Topounit elevation + real(r8) :: max_tpuElv ! Maximum topounit elevation for calculating elevation range + real(r8) :: min_tpuElv ! Minimum topounit elevation for calculating elevation range + real(r8) :: crnt_temp_t ! Current downscaled topounit temperature + real(r8) :: temp_r ! Temporary topounit rainfall + real(r8) :: temp_s ! Temporary topounit snowfall + real(r8) :: t_th ! Temperature threshold for snowfall + real(r8) :: Ta_th1 ! Temperature at 0.5 Celsius in K 273.65 + real(r8) :: Ta_th2 ! Temperature at 2.0 Celsius in K 275.15 + real(r8) :: Ta_th3 ! Temperature at 2.5 Celsius in K 275.65 + real(r8) :: tmp_Snow_frc ! Current snow fraction + + real(r8) :: e + real(r8) :: qvsat + + real(r8) :: sum_qbot_g ! weighted sum of column-level lwrad + real(r8) :: sum_wtsq_g ! sum of weights that contribute to sum_lwrad_g + real(r8) :: qbot_norm_g ! normalization factors + real(r8) :: sum_lwrad_g ! weighted sum of column-level lwrad + real(r8) :: sum_wtslw_g ! sum of weights that contribute to sum_lwrad_g + real(r8) :: lwrad_norm_g ! normalization factors + real(r8) :: esatw ! saturation vapor pressure over water (Pa) + real(r8) :: esati ! saturation vapor pressure over ice (Pa) + real(r8) :: a0,a1,a2,a3,a4,a5,a6 ! coefficients for esat over water + real(r8) :: b0,b1,b2,b3,b4,b5,b6 ! coefficients for esat over ice + real(r8) :: tdc, temp ! Kelvins to Celcius function and its input + real(r8) :: vp ! water vapor pressure (Pa) + + real(r8), allocatable :: deltaRain(:) ! Deviation of subgrid rain from grid rain + real(r8), allocatable :: deltaSnow(:) ! Deviation of subgrid snow from grid snow + real(r8) :: deltaR ! Temporary deltaRain + real(r8) :: deltaS ! Temporary deltaSnow + real(r8) :: sum_of_hrise ! Sum of height rise of air parcel of all subgrids of a grid + real(r8) :: hrise ! Temporary height rise + real(r8) :: elvrnge ! Elevation range between lowest and highest tpu + real(r8) :: ave_elv + integer :: elv_flag ! Elevation flag to trac grids with +ve grid elevation and -ve tpu elevation + + integer :: uaflag = 0 + integer :: precip_dwn = 0 ! Used to turn on/off the downscaling of precipitation 0 = on; 1 = off + integer :: other_forcing_dwn = 0 ! Used to turn on/off the downscaling of other forcing 0 = on; 1 = off + + character(len=*), parameter :: subname = 'downscale_atm_forcing_to_topounit' + !---------------------------------------------------------------------------------------- + + ! Constants to compute vapor pressure + parameter (a0=6.107799961_r8 , a1=4.436518521e-01_r8, & + a2=1.428945805e-02_r8, a3=2.650648471e-04_r8, & + a4=3.031240396e-06_r8, a5=2.034080948e-08_r8, & + a6=6.136820929e-11_r8) + + parameter (b0=6.109177956_r8 , b1=5.034698970e-01_r8, & + b2=1.886013408e-02_r8, b3=4.176223716e-04_r8, & + b4=5.824720280e-06_r8, b5=4.838803174e-08_r8, & + b6=1.838826904e-10_r8) + + ! + ! function declarations + ! + tdc(temp) = min( 50._r8, max(-50._r8,(temp-SHR_CONST_TKFRZ)) ) ! Taken from lnd_import_export.F90 + esatw(temp) = 100._r8*(a0+temp*(a1+temp*(a2+temp*(a3+temp*(a4+temp*(a5+temp*a6)))))) ! Taken from lnd_import_export.F90 + esati(temp) = 100._r8*(b0+temp*(b1+temp*(b2+temp*(b3+temp*(b4+temp*(b5+temp*b6)))))) ! Taken from lnd_import_export.F90 + !----------------------------------------------------------------------- + ! Get required inputs + numt_pg = grc_pp%ntopounits(g) ! Number of topounits per grid + grdElv = grc_pp%elevation(g) ! Grid level sfc elevation + mxElv = grc_pp%MaxElevation(g) ! Maximum src elevation per grid obtained from the highest elevation topounit + uovern_t = atm2lnd_vars%forc_uovern(g) ! Froude Number + + snow_g = atm2lnd_vars%forc_snow_not_downscaled_grc(g) + rain_g = atm2lnd_vars%forc_rain_not_downscaled_grc(g) + + sum_qbot_g = 0._r8 + sum_wtsq_g = 0._r8 + sum_lwrad_g = 0._r8 + sum_wtslw_g = 0._r8 + + sum_of_hrise = 0._r8 + t_th = 273.15_r8 ! Freezing temperature in K + Ta_th1 = 273.65_r8 ! Lowest threshold for snow calculation + Ta_th2 = 275.15_r8 ! Middle threshold for rain/snow partitioning + Ta_th3 = 275.65_r8 ! Highest threshold for rain/snow partitioning + tmp_Snow_frc = 0._r8 ! Snow fraction + elv_flag = 0 + elvrnge = 0._r8 + ave_elv = 0._r8 + max_tpuElv = 0._r8 + min_tpuElv = 0._r8 + if (numt_pg > 1) then !downscaling is done only if a grid has more than 1 topounits + if (precip_downscaling_method == 'FNM') then + allocate(deltaRain(numt_pg)) + deltaRain(:) = 0._r8 + allocate(deltaSnow(numt_pg)) + deltaSnow(:) = 0._r8 + hrise = 0._r8 + end if + + ! calculate elevation range and track grids with +ve elevation but have -ve tpu elevation + max_tpuElv = -100000._r8 + min_tpuElv = 100000._r8 + do t = grc_pp%topi(g), grc_pp%topf(g) ! Check occurence of grid elevation is +ve while tpu elevation is -ve + topoElv = top_pp%elevation(t) + if (topoElv > max_tpuElv) then + max_tpuElv = topoElv + end if + if (topoElv < min_tpuElv) then + min_tpuElv = topoElv + end if + end do + elvrnge = max_tpuElv - min_tpuElv + + do t = grc_pp%topi(g), grc_pp%topf(g) + t2 = t - grc_pp%topi(g) + 1 + topoElv = top_pp%elevation(t) ! Topounit sfc elevation + + ! Downscale precipitation + if (mxElv == 0._r8 .or. precip_dwn == 1) then ! avoid dividing by 0 + top_af%rain(t) = rain_g + top_af%snow(t) = snow_g + else + if (precip_downscaling_method == 'FNM') then + call downscale_precip_to_topounit_FNM(mxElv,uovern_t,grdElv,topoElv,rain_g,snow_g,deltaR,deltaS,hrise) !Use FNM method + deltaRain(t2) = deltaR + deltaSnow(t2) = deltaS + sum_of_hrise = sum_of_hrise + hrise + else + call downscale_precip_to_topounit_ERMM(t,mxElv,grdElv,topoElv,rain_g, snow_g,elv_flag,elvrnge) ! Use ERMM method + end if + end if + + ! Downscale other fluxes + if (other_forcing_dwn == 1) then ! flag to turn on or off downscaling of other forcing + top_af%rain(t) = rain_g + top_af%snow(t) = snow_g + top_af%lwrad(t) = atm2lnd_vars%forc_lwrad_not_downscaled_grc(g) + ! Update top_as + top_as%tbot(t) = atm2lnd_vars%forc_t_not_downscaled_grc(g) ! forc_txy Atm state K + top_as%thbot(t) = atm2lnd_vars%forc_th_not_downscaled_grc(g) ! forc_thxy Atm state K + top_as%pbot(t) = atm2lnd_vars%forc_pbot_not_downscaled_grc(g) ! ptcmxy Atm state Pa + top_as%qbot(t) = atm2lnd_vars%forc_q_not_downscaled_grc(g) ! forc_qxy Atm state kg/kg + top_as%ubot(t) = atm2lnd_vars%forc_u_grc(g) ! forc_uxy Atm state m/s + top_as%vbot(t) = atm2lnd_vars%forc_v_grc(g) ! forc_vxy Atm state m/s + top_as%zbot(t) = atm2lnd_vars%forc_hgt_grc(g) ! zgcmxy Atm state m + + ! assign the state forcing fields derived from other inputs + ! Horizontal windspeed (m/s) + top_as%windbot(t) = sqrt(top_as%ubot(t)**2 + top_as%vbot(t)**2) + ! Relative humidity (percent) + if (top_as%tbot(t) > SHR_CONST_TKFRZ) then + e = esatw(tdc(top_as%tbot(t))) + else + e = esati(tdc(top_as%tbot(t))) + end if + qvsat = 0.622_r8*e / (top_as%pbot(t) - 0.378_r8*e) + top_as%rhbot(t) = 100.0_r8*(top_as%qbot(t) / qvsat) + ! partial pressure of oxygen (Pa) + top_as%po2bot(t) = o2_molar_const * top_as%pbot(t) + ! air density (kg/m**3) - uses a temporary calculation + ! of water vapor pressure (Pa) + vp = top_as%qbot(t) * top_as%pbot(t) / (0.622_r8 + 0.378_r8 * top_as%qbot(t)) + top_as%rhobot(t) = (top_as%pbot(t) - 0.378_r8 * vp) / (rair * top_as%tbot(t)) + + top_af%solad(t,2) = atm2lnd_vars%forc_solad_grc(g,2) + top_af%solad(t,1) = atm2lnd_vars%forc_solad_grc(g,1) + top_af%solai(t,2) = atm2lnd_vars%forc_solai_grc(g,2) + top_af%solai(t,1) = atm2lnd_vars%forc_solai_grc(g,1) + ! derived flux forcings + top_af%solar(t) = top_af%solad(t,2) + top_af%solad(t,1) + & + top_af%solai(t,2) + top_af%solai(t,1) + else + call downscale_atm_state_to_topounit_cpl_bypass(g, t, atm2lnd_vars) + call downscale_longwave_to_topounit_cpl_bypass(g, t, atm2lnd_vars) + top_as%ubot(t) = atm2lnd_vars%forc_u_grc(g) ! forc_uxy Atm state m/s + top_as%vbot(t) = atm2lnd_vars%forc_v_grc(g) ! forc_vxy Atm state m/s + top_as%zbot(t) = atm2lnd_vars%forc_hgt_grc(g) ! zgcmxy Atm state m + + sum_qbot_g = sum_qbot_g + top_pp%wtgcell(t)*top_as%qbot(t) + sum_wtsq_g = sum_wtsq_g + top_pp%wtgcell(t) + + ! assign the state forcing fields derived from other inputs + ! Horizontal windspeed (m/s) + top_as%windbot(t) = sqrt(top_as%ubot(t)**2 + top_as%vbot(t)**2) + ! partial pressure of oxygen (Pa) + top_as%po2bot(t) = o2_molar_const * top_as%pbot(t) + ! air density (kg/m**3) - uses a temporary calculation + ! of water vapor pressure (Pa) + vp = top_as%qbot(t) * top_as%pbot(t) / (0.622_r8 + 0.378_r8 * top_as%qbot(t)) + top_as%rhobot(t) = (top_as%pbot(t) - 0.378_r8 * vp) / (rair * top_as%tbot(t)) + top_af%solad(t,2) = atm2lnd_vars%forc_solad_grc(g,2) + top_af%solad(t,1) = atm2lnd_vars%forc_solad_grc(g,1) + top_af%solai(t,2) = atm2lnd_vars%forc_solai_grc(g,2) + top_af%solai(t,1) = atm2lnd_vars%forc_solai_grc(g,1) + ! derived flux forcings + top_af%solar(t) = top_af%solad(t,2) + top_af%solad(t,1) + & + top_af%solai(t,2) + top_af%solai(t,1) + + ! Keep track of the gridcell-level weighted sum for later normalization. + ! + ! This gridcell-level weighted sum just includes points for which we do the + ! downscaling (e.g., glc_mec points). Thus the contributing weights + ! generally do not add to 1. So to do the normalization properly, we also + ! need to keep track of the weights that have contributed to this sum. + sum_lwrad_g = sum_lwrad_g + top_pp%wtgcell(t)*top_af%lwrad(t) + sum_wtslw_g = sum_wtslw_g + top_pp%wtgcell(t) + end if + end do + if (precip_downscaling_method == 'FNM') then + do t = grc_pp%topi(g), grc_pp%topf(g) + t2 = t - grc_pp%topi(g) + 1 + if (mxElv == 0.) then ! avoid dividing by 0 + top_af%rain(t) = rain_g + top_af%snow(t) = snow_g + else + top_af%rain(t) = rain_g + (deltaRain(t2) - (rain_g/mxElv)*(sum_of_hrise/numt_pg)) + top_af%snow(t) = snow_g + (deltaSnow(t2) - (snow_g/mxElv)*(sum_of_hrise/numt_pg)) + end if + + end do + deallocate(deltaRain) + deallocate(deltaSnow) + end if + + ! Precipitation partitioning using simple method following Jordan (1991) + do t = grc_pp%topi(g), grc_pp%topf(g) + crnt_temp_t = top_as%tbot(t) + if (crnt_temp_t > Ta_th3) then ! No snow or all rain + temp_r = top_af%rain(t) + top_af%snow(t) + temp_s = 0._r8 + else if (crnt_temp_t >= Ta_th2 .and. crnt_temp_t <= Ta_th3) then + temp_s = (top_af%snow(t) + top_af%rain(t))*0.6_r8 ! 0.6 fraction of the total precip is snow + temp_r = (top_af%snow(t) + top_af%rain(t)) - temp_s + else if (crnt_temp_t < Ta_th2 .and. crnt_temp_t > Ta_th1) then + tmp_Snow_frc = (crnt_temp_t-Ta_th2)*(0.4_r8/(Ta_th1-Ta_th2))+0.6_r8 ! Snow fraction value 1-0.6 = 0.4 snowfrc is 1 at t = Ta_th1 + temp_s = (top_af%snow(t) + top_af%rain(t))*tmp_Snow_frc ! 0.6 is snow fraction at t = Ta_th2 + temp_r = (top_af%snow(t) + top_af%rain(t)) - temp_s + else ! crnt_temp_t <= Ta_th1 ==> all snow + temp_s = top_af%snow(t) + top_af%rain(t) + temp_r = 0._r8 + + end if + + top_af%rain(t) = temp_r + top_af%snow(t) = temp_s + + end do + + else !grid has a single topounit + ! update top_af using grid level values + t = grc_pp%topi(g) + top_af%rain(t) = rain_g + top_af%snow(t) = snow_g + top_af%lwrad(t) = atm2lnd_vars%forc_lwrad_not_downscaled_grc(g) + + ! Update top_as + top_as%tbot(t) = atm2lnd_vars%forc_t_not_downscaled_grc(g) ! forc_txy Atm state K + top_as%thbot(t) = atm2lnd_vars%forc_th_not_downscaled_grc(g) ! forc_thxy Atm state K + top_as%pbot(t) = atm2lnd_vars%forc_pbot_not_downscaled_grc(g) ! ptcmxy Atm state Pa + top_as%qbot(t) = atm2lnd_vars%forc_q_not_downscaled_grc(g) ! forc_qxy Atm state kg/kg + top_as%ubot(t) = atm2lnd_vars%forc_u_grc(g) ! forc_uxy Atm state m/s + top_as%vbot(t) = atm2lnd_vars%forc_v_grc(g) ! forc_vxy Atm state m/s + top_as%zbot(t) = atm2lnd_vars%forc_hgt_grc(g) ! zgcmxy Atm state m + + ! assign the state forcing fields derived from other inputs + ! Horizontal windspeed (m/s) + top_as%windbot(t) = sqrt(top_as%ubot(t)**2 + top_as%vbot(t)**2) + if (atm_gustiness) then + top_as%windbot(t) = sqrt(top_as%windbot(t)**2 + top_as%ugust(t)**2) + end if + ! Relative humidity (percent) + if (top_as%tbot(t) > SHR_CONST_TKFRZ) then + e = esatw(tdc(top_as%tbot(t))) + else + e = esati(tdc(top_as%tbot(t))) + end if + qvsat = 0.622_r8*e / (top_as%pbot(t) - 0.378_r8*e) + top_as%rhbot(t) = 100.0_r8*(top_as%qbot(t) / qvsat) + ! partial pressure of oxygen (Pa) + top_as%po2bot(t) = o2_molar_const * top_as%pbot(t) + ! air density (kg/m**3) - uses a temporary calculation + ! of water vapor pressure (Pa) + vp = top_as%qbot(t) * top_as%pbot(t) / (0.622_r8 + 0.378_r8 * top_as%qbot(t)) + top_as%rhobot(t) = (top_as%pbot(t) - 0.378_r8 * vp) / (rair * top_as%tbot(t)) + + top_af%solad(t,2) = atm2lnd_vars%forc_solad_grc(g,2) + top_af%solad(t,1) = atm2lnd_vars%forc_solad_grc(g,1) + top_af%solai(t,2) = atm2lnd_vars%forc_solai_grc(g,2) + top_af%solai(t,1) = atm2lnd_vars%forc_solai_grc(g,1) + ! derived flux forcings + top_af%solar(t) = top_af%solad(t,2) + top_af%solad(t,1) + & + top_af%solai(t,2) + top_af%solai(t,1) + + end if + + if (numt_pg > 1) then + + ! Normalize forc_qbot to conserve energy + + call build_normalization(orig_field=atm2lnd_vars%forc_q_not_downscaled_grc(g), & + sum_field=sum_qbot_g, sum_wts=sum_wtsq_g, norms=qbot_norm_g) + + do t = grc_pp%topi(g), grc_pp%topf(g) + top_as%qbot(t) = top_as%qbot(t) * qbot_norm_g + + ! Relative humidity (percent) + if (top_as%tbot(t) > SHR_CONST_TKFRZ) then + e = esatw(tdc(top_as%tbot(t))) + else + e = esati(tdc(top_as%tbot(t))) + end if + qvsat = 0.622_r8*e / (top_as%pbot(t) - 0.378_r8*e) + top_as%rhbot(t) = 100.0_r8*(top_as%qbot(t) / qvsat) + ! partial pressure of oxygen (Pa) + top_as%po2bot(t) = o2_molar_const * top_as%pbot(t) + ! air density (kg/m**3) - uses a temporary calculation + ! of water vapor pressure (Pa) + vp = top_as%qbot(t) * top_as%pbot(t) / (0.622_r8 + 0.378_r8 * top_as%qbot(t)) + top_as%rhobot(t) = (top_as%pbot(t) - 0.378_r8 * vp) / (rair * top_as%tbot(t)) + + end do + + + ! Normalize forc_lwrad_c(c) to conserve energy + + call build_normalization(orig_field=atm2lnd_vars%forc_lwrad_not_downscaled_grc(g), & + sum_field=sum_lwrad_g, sum_wts=sum_wtslw_g, norms=lwrad_norm_g) + + do t = grc_pp%topi(g), grc_pp%topf(g) + top_af%lwrad(t) = top_af%lwrad(t) * lwrad_norm_g + end do + + end if + + end subroutine downscale_atm_forcing_to_topounit_cpl_bypass + + !----------------------------------------------------------------------- + ! Downscale other atmospheric state variables + !----------------------------------------------------------------------- + subroutine downscale_atm_state_to_topounit_cpl_bypass(g, t, atm2lnd_vars) + ! + ! !DESCRIPTION: + ! Downscale atmospheric forcing fields from gridcell to topounit + ! + ! Downscaling is done over topounits. + ! + ! !USES: + use elm_time_manager, only : get_nstep + use elm_varcon , only : rair, cpair, grav, lapse_glcmec + use elm_varcon , only : glcmec_rain_snow_threshold + use landunit_varcon , only : istice_mec + use elm_varctl , only : glcmec_downscale_rain_snow_convert + use domainMod , only : ldomain + use QsatMod , only : Qsat + ! + ! !ARGUMENTS: + integer , intent(in) :: g + integer , intent(in) :: t + type(atm2lnd_type) , intent(in) :: atm2lnd_vars + ! + ! !LOCAL VARIABLES: + integer :: l, c, fc ! indices + integer :: clo, cc + integer :: nstep + + ! temporaries for topo downscaling + real(r8) :: hsurf_g,hsurf_t,Hbot + real(r8) :: zbot_g, tbot_g, pbot_g, thbot_g, qbot_g, qs_g, es_g + real(r8) :: zbot_t, tbot_t, pbot_t, thbot_t, qbot_t, qs_t, es_t + real(r8) :: egcm_t, rhos_t + real(r8) :: dum1, dum2 + + character(len=*), parameter :: subname = 'downscale_atm_state_to_topounit' + !----------------------------------------------------------------------- + + nstep = get_nstep() + + ! Downscale forc_t, forc_th, forc_q, forc_pbot, and forc_rho to columns. + ! For glacier_mec columns the downscaling is based on surface elevation. + ! For other columns the downscaling is a simple copy (above). + + ! This is a simple downscaling procedure + ! Note that forc_hgt, forc_u, and forc_v are not downscaled. + + hsurf_g = grc_pp%elevation(g) ! gridcell sfc elevation + hsurf_t = top_pp%elevation(t) ! topounit sfc elevation + tbot_g = atm2lnd_vars%forc_t_not_downscaled_grc(g) ! atm sfc temp + thbot_g = atm2lnd_vars%forc_th_not_downscaled_grc(g) ! atm sfc pot temp + qbot_g = atm2lnd_vars%forc_q_not_downscaled_grc(g) ! atm sfc spec humid + pbot_g = atm2lnd_vars%forc_pbot_not_downscaled_grc(g) ! atm sfc pressure + zbot_g = atm2lnd_vars%forc_hgt_grc(g) ! atm ref height + + zbot_t = zbot_g + tbot_t = tbot_g-lapse_glcmec*(hsurf_t-hsurf_g) ! sfc temp for column + + Hbot = rair*0.5_r8*(tbot_g+tbot_t)/grav ! scale ht at avg temp + pbot_t = pbot_g*exp(-(hsurf_t-hsurf_g)/Hbot) ! column sfc press + + ! Derivation of potential temperature calculation: + ! + ! The textbook definition would be: + ! thbot_c = tbot_c * (p0/pbot_c)^(rair/cpair) + ! + ! Note that pressure is related to scale height as: + ! pbot_c = p0 * exp(-zbot_c/H) + ! + ! Using Hbot in place of H, we get: + ! pbot_c = p0 * exp(-zbot_c/Hbot) + ! + ! Plugging this in to the textbook definition, then manipulating, we get: + ! thbot_c = tbot_c * (p0/(p0*exp(-zbot_c/Hbot)))^(rair/cpair) + ! = tbot_c * (1/exp(-zbot_c/Hbot))^(rair/cpair) + ! = tbot_c * (exp(zbot_c/Hbot))^(rair/cpair) + ! = tbot_c * exp((zbot_c/Hbot) * (rair/cpair)) + + thbot_t= tbot_t*exp((zbot_t/Hbot)*(rair/cpair)) ! pot temp calc + + call Qsat(tbot_g,pbot_g,es_g,dum1,qs_g,dum2) + call Qsat(tbot_t,pbot_t,es_t,dum1,qs_t,dum2) + + qbot_t = qbot_g*(qs_t/qs_g) + egcm_t = qbot_t*pbot_t/(0.622_r8+0.378_r8*qbot_t) + rhos_t = (pbot_t-0.378_r8*egcm_t) / (rair*tbot_t) + + top_as%tbot(t) = tbot_t + top_as%thbot(t) = thbot_t + top_as%qbot(t) = qbot_t + top_as%pbot(t) = pbot_t + +! call check_downscale_consistency(bounds, atm2lnd_vars) + + end subroutine downscale_atm_state_to_topounit_cpl_bypass + + !------------------------------------------------------- + ! Downscale longwave radiation place holder + subroutine downscale_longwave_to_topounit_cpl_bypass(g, t, atm2lnd_vars) + + ! !DESCRIPTION: + ! Downscale longwave radiation from gridcell to column + ! Must be done AFTER temperature downscaling + + ! !USES: + use elm_time_manager, only : get_nstep + use domainMod , only : ldomain + use landunit_varcon , only : istice_mec + use elm_varcon , only : lapse_glcmec + use elm_varctl , only : glcmec_downscale_longwave + + ! !ARGUMENTS: + integer , intent(in) :: g + integer , intent(in) :: t + type(atm2lnd_type) , intent(in) :: atm2lnd_vars + + ! !LOCAL VARIABLES: + integer :: c,l,fc ! indices + integer :: nstep + real(r8) :: hsurf_t ! column-level elevation (m) + real(r8) :: hsurf_g ! gridcell-level elevation (m) + + real(r8) :: tair_g ! original gridcell mean air temperature + real(r8) :: tair_t ! downscaled topounit air temperature + real(r8) :: tsfc_g ! original gridcell surface temperature + real(r8) :: tsfc_t ! downscaled topounit surface temperature + real(r8) :: lwrad_g ! original gridcell mean LW radiation + real(r8) :: lwrad_t ! downscaled topounit LW radiation + real(r8) :: newsum_lwrad_g ! weighted sum of column-level lwrad after normalization + + character(len=*), parameter :: subname = 'downscale_longwave_to_topounit' + !----------------------------------------------------------------------- + + nstep = get_nstep() + + ! Do the downscaling + hsurf_g = grc_pp%elevation(g) + hsurf_t = top_pp%elevation(t) + + ! Here we assume that deltaLW = (dLW/dT)*(dT/dz)*deltaz + ! We get dLW/dT = 4*eps*sigma*T^3 = 4*LW/T from the Stefan-Boltzmann law, + ! evaluated at the mean temp. + ! We assume the same temperature lapse rate as above. + + tair_g = atm2lnd_vars%forc_t_not_downscaled_grc(g) + tair_t = top_as%tbot(t) + lwrad_g = atm2lnd_vars%forc_lwrad_not_downscaled_grc(g) + top_af%lwrad(t) = lwrad_g - & + 4.0_r8 * lwrad_g/(0.5_r8*(tair_t+tair_g)) * & + lapse_glcmec * (hsurf_t - hsurf_g) + + end subroutine downscale_longwave_to_topounit_cpl_bypass + end module lnd_downscale_atm_forcing diff --git a/components/elm/src/cpl/lnd_import_export.F90 b/components/elm/src/cpl/lnd_import_export.F90 index ad81846196d0..27c01e668184 100644 --- a/components/elm/src/cpl/lnd_import_export.F90 +++ b/components/elm/src/cpl/lnd_import_export.F90 @@ -1030,52 +1030,68 @@ subroutine lnd_import( bounds, x2l, atm2lnd_vars, glc2lnd_vars, lnd2atm_vars) end if end if - !set the topounit-level atmospheric state and flux forcings (bypass mode) + ! Adding topographic downscaling capability within CPL_BYPASS code block + ! PET, 7/12/2024 + if (use_atm_downscaling_to_topunit) then + atm2lnd_vars%forc_uovern = x2l(index_x2l_Sa_uovern,i) + atm2lnd_vars%forc_rain_not_downscaled_grc = forc_rainc + forc_rainl + atm2lnd_vars%forc_snow_not_downscaled_grc = forc_snowc + forc_snowl + + if(atm_gustiness) then + call endrun("Error: atm_gustiness not yet supported with multiple topounits (in CPL_BYPASS)") + end if + do topo = grc_pp%topi(g) , grc_pp%topf(g) + top_as%ugust(topo) = 0._r8 + end do + + call downscale_atm_forcing_to_topounit_cpl_bypass(g, atm2lnd_vars, lnd2atm_vars) + else + do topo = grc_pp%topi(g), grc_pp%topf(g) + top_as%tbot(topo) = atm2lnd_vars%forc_t_not_downscaled_grc(g) ! forc_txy Atm state K + top_as%thbot(topo) = atm2lnd_vars%forc_th_not_downscaled_grc(g) ! forc_thxy Atm state K + top_as%pbot(topo) = atm2lnd_vars%forc_pbot_not_downscaled_grc(g) ! ptcmxy Atm state Pa + top_as%qbot(topo) = atm2lnd_vars%forc_q_not_downscaled_grc(g) ! forc_qxy Atm state kg/kg + top_as%ubot(topo) = atm2lnd_vars%forc_u_grc(g) ! forc_uxy Atm state m/s + top_as%vbot(topo) = atm2lnd_vars%forc_v_grc(g) ! forc_vxy Atm state m/s + top_as%zbot(topo) = atm2lnd_vars%forc_hgt_grc(g) ! zgcmxy Atm state m + top_as%windbot(topo) = sqrt(top_as%ubot(topo)**2 + top_as%vbot(topo)**2) + ! Relative humidity (percent) + if (top_as%tbot(topo) > SHR_CONST_TKFRZ) then + e = esatw(tdc(top_as%tbot(topo))) + else + e = esati(tdc(top_as%tbot(topo))) + end if + qsat = 0.622_r8*e / (top_as%pbot(topo) - 0.378_r8*e) + top_as%rhbot(topo) = 100.0_r8*(top_as%qbot(topo) / qsat) + ! partial pressure of oxygen (Pa) + top_as%po2bot(topo) = o2_molar_const * top_as%pbot(topo) + ! air density (kg/m**3) - uses a temporary calculation of water vapor pressure (Pa) + vp = top_as%qbot(topo) * top_as%pbot(topo) / (0.622_r8 + 0.378_r8 * top_as%qbot(topo)) + top_as%rhobot(topo) = (top_as%pbot(topo) - 0.378_r8 * vp) / (rair * top_as%tbot(topo)) + top_af%rain(topo) = forc_rainc + forc_rainl ! sum of convective and large-scale rain + top_af%snow(topo) = forc_snowc + forc_snowl ! sum of convective and large-scale snow + top_af%solad(topo,2) = atm2lnd_vars%forc_solad_grc(g,2) ! forc_sollxy Atm flux W/m^2 + top_af%solad(topo,1) = atm2lnd_vars%forc_solad_grc(g,1) ! forc_solsxy Atm flux W/m^2 + top_af%solai(topo,2) = atm2lnd_vars%forc_solai_grc(g,2) ! forc_solldxy Atm flux W/m^2 + top_af%solai(topo,1) = atm2lnd_vars%forc_solai_grc(g,1) ! forc_solsdxy Atm flux W/m^2 + top_af%lwrad(topo) = atm2lnd_vars%forc_lwrad_not_downscaled_grc(g) ! flwdsxy Atm flux W/m^2 + ! derived flux forcings + top_af%solar(topo) = top_af%solad(topo,2) + top_af%solad(topo,1) + & + top_af%solai(topo,2) + top_af%solai(topo,1) + end do + end if + + !set the topounit-level atmospheric variables that are not handled in downscaling code do topo = grc_pp%topi(g), grc_pp%topf(g) ! first, all the state forcings - top_as%tbot(topo) = atm2lnd_vars%forc_t_not_downscaled_grc(g) ! forc_txy Atm state K - top_as%thbot(topo) = atm2lnd_vars%forc_th_not_downscaled_grc(g) ! forc_thxy Atm state K - top_as%pbot(topo) = atm2lnd_vars%forc_pbot_not_downscaled_grc(g) ! ptcmxy Atm state Pa - top_as%qbot(topo) = atm2lnd_vars%forc_q_not_downscaled_grc(g) ! forc_qxy Atm state kg/kg - top_as%ubot(topo) = atm2lnd_vars%forc_u_grc(g) ! forc_uxy Atm state m/s - top_as%vbot(topo) = atm2lnd_vars%forc_v_grc(g) ! forc_vxy Atm state m/s - if (implicit_stress) then + if (implicit_stress) then top_as%wsresp(topo) = 0._r8 ! Atm state m/s/Pa top_as%tau_est(topo) = 0._r8 ! Atm state Pa end if top_as%ugust(topo) = 0._r8 ! Atm state m/s - top_as%zbot(topo) = atm2lnd_vars%forc_hgt_grc(g) ! zgcmxy Atm state m - ! assign the state forcing fields derived from other inputs - ! Horizontal windspeed (m/s) - top_as%windbot(topo) = sqrt(top_as%ubot(topo)**2 + top_as%vbot(topo)**2) if (atm_gustiness) then top_as%windbot(topo) = sqrt(top_as%windbot(topo)**2 + top_as%ugust(topo)**2) end if - ! Relative humidity (percent) - if (top_as%tbot(topo) > SHR_CONST_TKFRZ) then - e = esatw(tdc(top_as%tbot(topo))) - else - e = esati(tdc(top_as%tbot(topo))) - end if - qsat = 0.622_r8*e / (top_as%pbot(topo) - 0.378_r8*e) - top_as%rhbot(topo) = 100.0_r8*(top_as%qbot(topo) / qsat) - ! partial pressure of oxygen (Pa) - top_as%po2bot(topo) = o2_molar_const * top_as%pbot(topo) - ! air density (kg/m**3) - uses a temporary calculation of water vapor pressure (Pa) - vp = top_as%qbot(topo) * top_as%pbot(topo) / (0.622_r8 + 0.378_r8 * top_as%qbot(topo)) - top_as%rhobot(topo) = (top_as%pbot(topo) - 0.378_r8 * vp) / (rair * top_as%tbot(topo)) - - ! second, all the flux forcings - top_af%rain(topo) = forc_rainc + forc_rainl ! sum of convective and large-scale rain - top_af%snow(topo) = forc_snowc + forc_snowl ! sum of convective and large-scale snow - top_af%solad(topo,2) = atm2lnd_vars%forc_solad_grc(g,2) ! forc_sollxy Atm flux W/m^2 - top_af%solad(topo,1) = atm2lnd_vars%forc_solad_grc(g,1) ! forc_solsxy Atm flux W/m^2 - top_af%solai(topo,2) = atm2lnd_vars%forc_solai_grc(g,2) ! forc_solldxy Atm flux W/m^2 - top_af%solai(topo,1) = atm2lnd_vars%forc_solai_grc(g,1) ! forc_solsdxy Atm flux W/m^2 - top_af%lwrad(topo) = atm2lnd_vars%forc_lwrad_not_downscaled_grc(g) ! flwdsxy Atm flux W/m^2 - ! derived flux forcings - top_af%solar(topo) = top_af%solad(topo,2) + top_af%solad(topo,1) + & - top_af%solai(topo,2) + top_af%solai(topo,1) end do !----------------------------------------------------------------------------------------------------- diff --git a/components/elm/src/data_types/ColumnDataType.F90 b/components/elm/src/data_types/ColumnDataType.F90 index d0a4a10cd3d6..ba278d1fc467 100644 --- a/components/elm/src/data_types/ColumnDataType.F90 +++ b/components/elm/src/data_types/ColumnDataType.F90 @@ -502,6 +502,9 @@ module ColumnDataType real(r8), pointer :: qflx_glcice (:) => null() ! net flux of new glacial ice (growth - melt) (mm H2O/s), passed to GLC real(r8), pointer :: qflx_glcice_frz (:) => null() ! ice growth (positive definite) (mm H2O/s) real(r8), pointer :: qflx_glcice_melt (:) => null() ! ice melt (positive definite) (mm H2O/s) + real(r8), pointer :: qflx_glcice_diag (:) => null() ! net flux of new glacial ice (growth - melt) (mm H2O/s), passed to GLC + real(r8), pointer :: qflx_glcice_frz_diag (:) => null() ! ice growth (positive definite) (mm H2O/s) + real(r8), pointer :: qflx_glcice_melt_diag(:) => null() ! ice melt (positive definite) (mm H2O/s) real(r8), pointer :: qflx_drain_vr (:,:) => null() ! liquid water lost as drainage (m /time step) real(r8), pointer :: qflx_h2osfc2topsoi (:) => null() ! liquid water coming from surface standing water top soil (mm H2O/s) real(r8), pointer :: qflx_snow2topsoi (:) => null() ! liquid water coming from residual snow to topsoil (mm H2O/s) @@ -5725,6 +5728,9 @@ subroutine col_wf_init(this, begc, endc) allocate(this%qflx_glcice (begc:endc)) ; this%qflx_glcice (:) = spval allocate(this%qflx_glcice_frz (begc:endc)) ; this%qflx_glcice_frz (:) = spval allocate(this%qflx_glcice_melt (begc:endc)) ; this%qflx_glcice_melt (:) = spval + allocate(this%qflx_glcice_diag (begc:endc)) ; this%qflx_glcice_diag (:) = spval + allocate(this%qflx_glcice_frz_diag (begc:endc)) ; this%qflx_glcice_frz_diag (:) = spval + allocate(this%qflx_glcice_melt_diag (begc:endc)) ; this%qflx_glcice_melt_diag(:) = spval allocate(this%qflx_drain_vr (begc:endc,1:nlevgrnd)) ; this%qflx_drain_vr (:,:) = spval allocate(this%qflx_h2osfc2topsoi (begc:endc)) ; this%qflx_h2osfc2topsoi (:) = spval allocate(this%qflx_snow2topsoi (begc:endc)) ; this%qflx_snow2topsoi (:) = spval @@ -5842,23 +5848,39 @@ subroutine col_wf_init(this, begc, endc) call hist_addfld1d (fname='QSNOFRZ', units='kg/m2/s', & avgflag='A', long_name='column-integrated snow freezing rate', & ptr_col=this%qflx_snofrz, set_lake=spval, c2l_scale_type='urbanf', default='inactive') - + if (create_glacier_mec_landunit) then - this%qflx_glcice(begc:endc) = spval - call hist_addfld1d (fname='QICE', units='mm/s', & - avgflag='A', long_name='ice growth/melt', & - ptr_col=this%qflx_glcice, l2g_scale_type='ice') - - this%qflx_glcice_frz(begc:endc) = spval - call hist_addfld1d (fname='QICE_FRZ', units='mm/s', & - avgflag='A', long_name='ice growth', & - ptr_col=this%qflx_glcice_frz, l2g_scale_type='ice') - - this%qflx_glcice_melt(begc:endc) = spval - call hist_addfld1d (fname='QICE_MELT', units='mm/s', & - avgflag='A', long_name='ice melt', & - ptr_col=this%qflx_glcice_melt, l2g_scale_type='ice') - endif + this%qflx_glcice(begc:endc) = spval + call hist_addfld1d (fname='QICE', units='mm/s', & + avgflag='A', long_name='ice growth/melt (with active GLC/MECs)', & + ptr_col=this%qflx_glcice, l2g_scale_type='ice') + + this%qflx_glcice_frz(begc:endc) = spval + call hist_addfld1d (fname='QICE_FRZ', units='mm/s', & + avgflag='A', long_name='ice growth (with active GLC/MECs)', & + ptr_col=this%qflx_glcice_frz, l2g_scale_type='ice') + + this%qflx_glcice_melt(begc:endc) = spval + call hist_addfld1d (fname='QICE_MELT', units='mm/s', & + avgflag='A', long_name='ice melt (with active GLC/MECs)', & + ptr_col=this%qflx_glcice_melt, l2g_scale_type='ice') + else + this%qflx_glcice_diag(begc:endc) = spval + call hist_addfld1d (fname='QICE', units='mm/s', & + avgflag='A', long_name='diagnostic ice growth/melt (no active GLC/MECs)', & + ptr_col=this%qflx_glcice_diag, l2g_scale_type='ice') + + this%qflx_glcice_frz_diag(begc:endc) = spval + call hist_addfld1d (fname='QICE_FRZ', units='mm/s', & + avgflag='A', long_name='diagnostic ice growth (no active GLC/MECs)', & + ptr_col=this%qflx_glcice_frz_diag, l2g_scale_type='ice') + + this%qflx_glcice_melt_diag(begc:endc) = spval + call hist_addfld1d (fname='QICE_MELT', units='mm/s', & + avgflag='A', long_name='diagnostic ice melt (no active GLC/MECs)', & + ptr_col=this%qflx_glcice_melt_diag, l2g_scale_type='ice') + end if + ! As defined here, snow_sources - snow_sinks will equal the change in h2osno at any ! given time step but only if there is at least one snow layer (for all landunits diff --git a/components/elm/src/data_types/VegetationPropertiesType.F90 b/components/elm/src/data_types/VegetationPropertiesType.F90 index 12f9a7b22ffa..d62167301cd4 100644 --- a/components/elm/src/data_types/VegetationPropertiesType.F90 +++ b/components/elm/src/data_types/VegetationPropertiesType.F90 @@ -43,7 +43,7 @@ module VegetationPropertiesType real(r8), pointer :: dsladlai (:) => null() ! dSLA/dLAI, projected area basis [m^2/gC] real(r8), pointer :: leafcn (:) => null() ! leaf C:N (gC/gN) real(r8), pointer :: flnr (:) => null() ! fraction of leaf N in the Rubisco enzyme (gN Rubisco / gN leaf) - real(r8), pointer :: woody (:) => null() ! binary flag for woody lifeform (1=woody, 0=not woody) + real(r8), pointer :: woody (:) => null() ! woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) real(r8), pointer :: lflitcn (:) => null() ! leaf litter C:N (gC/gN) real(r8), pointer :: frootcn (:) => null() ! fine root C:N (gC/gN) real(r8), pointer :: livewdcn (:) => null() ! live wood (phloem and ray parenchyma) C:N (gC/gN) @@ -115,7 +115,7 @@ module VegetationPropertiesType real(r8), pointer :: lamda_ptase => null()! critical value that incur biochemical production real(r8), pointer :: i_vc(:) => null() ! intercept of photosynthesis vcmax ~ leaf n content regression model real(r8), pointer :: s_vc(:) => null() ! slope of photosynthesis vcmax ~ leaf n content regression model - real(r8), pointer :: nsc_rtime(:) => null() ! non-structural carbon residence time + real(r8), pointer :: nsc_rtime(:) => null() ! non-structural carbon residence time real(r8), pointer :: pinit_beta1(:) => null() ! shaping parameter for P initialization real(r8), pointer :: pinit_beta2(:) => null() ! shaping parameter for P initialization real(r8), pointer :: alpha_nfix(:) => null() ! fraction of fixed N goes directly to plant @@ -151,6 +151,11 @@ module VegetationPropertiesType real(r8), pointer :: needleleaf(:) => null() !needleleaf or broadleaf real(r8), pointer :: nfixer(:) => null() !cablity of nitrogen fixation from atm. N2 + ! NGEE Arctic snow-vegetation interactions + real(r8), pointer :: bendresist(:) ! vegetation resistance to bending under snow loading, 0 to 1 (e.g., Liston and Hiemstra 2011) + real(r8), pointer :: vegshape(:) ! shape parameter to modify shrub burial by snow (1 = parabolic, 2 = hemispheric) + real(r8), pointer :: stocking(:) ! stocking density for pft (stems / hectare) + real(r8), pointer :: taper(:) ! ratio of height:radius_breast_height (woody vegetation allometry) contains procedure, public :: Init => veg_vp_init @@ -188,8 +193,10 @@ subroutine veg_vp_init(this) use pftvarcon , only : fnr, act25, kcha, koha, cpha, vcmaxha, jmaxha, tpuha use pftvarcon , only : lmrha, vcmaxhd, jmaxhd, tpuhd, lmrse, qe, theta_cj use pftvarcon , only : bbbopt, mbbopt, nstor, br_xr, tc_stress, lmrhd - ! new properties for flexible PFT + ! new properties for flexible PFT (NGEE Arctic IM4) use pftvarcon , only : climatezone, nonvascular, graminoid, iscft,needleleaf, nfixer + ! snow/vegetation interactions (NGEE Arctic IM3) + use pftvarcon , only : bendresist, stocking, vegshape, taper ! class (vegetation_properties_type) :: this @@ -322,6 +329,11 @@ subroutine veg_vp_init(this) allocate( this%needleleaf(0:numpft)) ; this%needleleaf(:) =spval allocate( this%nfixer(0:numpft)) ; this%nfixer(:) =spval ! ----------------------------------------------------------------------------------------------------------- + ! NGEE Arctic snow-vegetation interactions + allocate(this%bendresist(0:numpft)) ; this%bendresist(:) =spval + allocate(this%vegshape(0:numpft)) ; this%vegshape(:) =spval + allocate(this%stocking(0:numpft)) ; this%stocking(:) =spval + allocate(this%taper(0:numpft)) ; this%taper(:) =spval do m = 0,numpft @@ -472,6 +484,13 @@ subroutine veg_vp_init(this) this%lamda_ptase = lamda_ptase this%tc_stress = tc_stress + ! NGEE Arctic - snow/vegetation interactions + do m = 0, numpft ! RPF - move up to earlier pft loops? + this%bendresist(m) = bendresist(m) + this%vegshape(m) = vegshape(m) + this%stocking(m) = stocking(m) + this%taper(m) = taper(m) + end do end subroutine veg_vp_init end module VegetationPropertiesType diff --git a/components/elm/src/external_models/fates b/components/elm/src/external_models/fates index 1982b0032c3c..e3e7d2cd86a6 160000 --- a/components/elm/src/external_models/fates +++ b/components/elm/src/external_models/fates @@ -1 +1 @@ -Subproject commit 1982b0032c3cab6278892eccb85f643114ffb1af +Subproject commit e3e7d2cd86a66f8ca0e8f6dc4a823246a2bdb95b diff --git a/components/elm/src/external_models/sbetr b/components/elm/src/external_models/sbetr index 66260f4991d6..08d8a8184a60 160000 --- a/components/elm/src/external_models/sbetr +++ b/components/elm/src/external_models/sbetr @@ -1 +1 @@ -Subproject commit 66260f4991d61439d4cba92eb633590b09f97920 +Subproject commit 08d8a8184a605a23d4dce4f91a33d8f2bba29ae9 diff --git a/components/elm/src/main/atm2lndType.F90 b/components/elm/src/main/atm2lndType.F90 index a00140ab2dd6..f2facc776224 100644 --- a/components/elm/src/main/atm2lndType.F90 +++ b/components/elm/src/main/atm2lndType.F90 @@ -146,6 +146,10 @@ module atm2lndType real(r8) , pointer :: t_mo_patch (:) => null() ! patch 30-day average temperature (Kelvin) real(r8) , pointer :: t_mo_min_patch (:) => null() ! patch annual min of t_mo (Kelvin) + ! Needed for FNM precip downscaling, when used within CPL_BYPASS + real(r8), pointer :: forc_uovern (:) => null() ! Froude number (dimensionless) + + contains procedure, public :: Init @@ -310,6 +314,7 @@ subroutine InitAllocate(this, bounds) allocate(this%forc_ndep_nitr_grc (begg:endg)) ; this%forc_ndep_nitr_grc (:) = ival allocate(this%forc_soilph_grc (begg:endg)) ; this%forc_soilph_grc (:) = ival end if + allocate(this%forc_uovern (begg:endg)) ; this%forc_uovern (:) = ival end subroutine InitAllocate diff --git a/components/elm/src/main/controlMod.F90 b/components/elm/src/main/controlMod.F90 index 1d8f48d7cc7d..dc7fa975dbb4 100755 --- a/components/elm/src/main/controlMod.F90 +++ b/components/elm/src/main/controlMod.F90 @@ -55,6 +55,7 @@ module controlMod use elm_varctl , only: const_climate_hist use elm_varctl , only: use_top_solar_rad use elm_varctl , only: snow_shape, snicar_atm_type, use_dust_snow_internal_mixing + use EcosystemBalanceCheckMod, only: bgc_balance_check_tolerance => balance_check_tolerance ! ! !PUBLIC TYPES: @@ -181,6 +182,10 @@ subroutine control_init( ) namelist /elm_inparm/ & NFIX_PTASE_plant + ! BGC balance check + namelist /elm_inparm/ & + bgc_balance_check_tolerance + ! For experimental manipulations namelist /elm_inparm/ & startdate_add_temperature @@ -325,7 +330,7 @@ subroutine control_init( ) namelist /elm_mosart/ & lnd_rof_coupling_nstep - + namelist /elm_inparm/ & snow_shape, snicar_atm_type, use_dust_snow_internal_mixing @@ -796,6 +801,7 @@ subroutine control_spmd() call mpi_bcast (forest_fert_exp, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (ECA_Pconst_RGspin, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (NFIX_PTASE_plant, 1, MPI_LOGICAL, 0, mpicom, ier) + call mpi_bcast (bgc_balance_check_tolerance, 1, MPI_REAL8, 0, mpicom, ier) call mpi_bcast (use_pheno_flux_limiter, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (startdate_add_temperature, 1, MPI_CHARACTER, 0, mpicom, ier) call mpi_bcast (startdate_add_co2, 1, MPI_CHARACTER, 0, mpicom, ier) diff --git a/components/elm/src/main/elm_driver.F90 b/components/elm/src/main/elm_driver.F90 index 51f08bf235c3..18ecae88e37f 100644 --- a/components/elm/src/main/elm_driver.F90 +++ b/components/elm/src/main/elm_driver.F90 @@ -1604,6 +1604,8 @@ subroutine elm_drv_init(bounds, & qflx_glcice => col_wf%qflx_glcice , & ! Output: [real(r8) (:) ] flux of new glacier ice (mm H2O/s) [+ = ice grows] + qflx_glcice_diag => col_wf%qflx_glcice_diag , & ! Output: [real(r8) (:) ] flux of new glacier ice (mm H2O/s) [+ = ice grows] + eflx_bot => col_ef%eflx_bot , & ! Output: [real(r8) (:) ] heat flux from beneath soil/ice column (W/m**2) cisun_z => photosyns_vars%cisun_z_patch , & ! Output: [real(r8) (:) ] intracellular sunlit leaf CO2 (Pa) @@ -1637,6 +1639,7 @@ subroutine elm_drv_init(bounds, & ! Initialize qflx_glcice everywhere, to zero. qflx_glcice(c) = 0._r8 + qflx_glcice_diag(c) = 0._r8 end do diff --git a/components/elm/src/main/elm_varctl.F90 b/components/elm/src/main/elm_varctl.F90 index fb9ca0dbd24e..69a3209e52fe 100644 --- a/components/elm/src/main/elm_varctl.F90 +++ b/components/elm/src/main/elm_varctl.F90 @@ -221,7 +221,7 @@ module elm_varctl logical, public :: use_fates = .false. ! true => use ED integer, public :: fates_spitfire_mode = 0 ! 0 for no fire; 1 for constant ignitions - character(len=13), public :: fates_harvest_mode = '' ! five different harvest modes; see namelist_definitions + character(len=256), public :: fates_harvest_mode = '' ! five different harvest modes; see namelist_definitions logical, public :: use_fates_fixed_biogeog = .false. ! true => use fixed biogeography mode logical, public :: use_fates_planthydro = .false. ! true => turn on fates hydro logical, public :: use_fates_cohort_age_tracking = .false. ! true => turn on cohort age tracking diff --git a/components/elm/src/main/elmfates_interfaceMod.F90 b/components/elm/src/main/elmfates_interfaceMod.F90 index a20444d67582..e7134a580fdc 100644 --- a/components/elm/src/main/elmfates_interfaceMod.F90 +++ b/components/elm/src/main/elmfates_interfaceMod.F90 @@ -3517,7 +3517,7 @@ subroutine hlm_bounds_to_fates_bounds(hlm, fates) use FatesInterfaceTypesMod, only : nlevage_fates => nlevage use FatesInterfaceTypesMod, only : nlevheight_fates => nlevheight use FatesInterfaceTypesMod, only : nlevdamage_fates => nlevdamage - use FatesLitterMod, only : nfsc_fates => nfsc + use FatesFuelClassesMod, only : nfc_fates => num_fuel_classes use FatesLitterMod, only : ncwd_fates => ncwd use EDParamsMod, only : nlevleaf_fates => nlevleaf use EDParamsMod, only : nclmax_fates => nclmax @@ -3555,7 +3555,7 @@ subroutine hlm_bounds_to_fates_bounds(hlm, fates) fates%sizeage_class_end = nlevsclass_fates * nlevage_fates fates%fuel_begin = 1 - fates%fuel_end = nfsc_fates + fates%fuel_end = nfc_fates fates%cdpf_begin = 1 fates%cdpf_end = nlevdamage_fates * numpft_fates * nlevsclass_fates @@ -3606,7 +3606,7 @@ subroutine hlm_bounds_to_fates_bounds(hlm, fates) fates%coage_class_end = nlevcoage fates%agefuel_begin = 1 - fates%agefuel_end = nlevage_fates * nfsc_fates + fates%agefuel_end = nlevage_fates * nfc_fates fates%landuse_begin = 1 fates%landuse_end = n_landuse_cats diff --git a/components/elm/src/main/histFileMod.F90 b/components/elm/src/main/histFileMod.F90 index 1b86c3a1618a..8de3f27c30ec 100644 --- a/components/elm/src/main/histFileMod.F90 +++ b/components/elm/src/main/histFileMod.F90 @@ -28,7 +28,7 @@ module histFileMod use FatesInterfaceTypesMod , only : nlevheight_fates => nlevheight use FatesInterfaceTypesMod , only : nlevdamage_fates => nlevdamage use FatesInterfaceTypesMod , only : nlevcoage - use FatesLitterMod , only : nfsc_fates => nfsc + use FatesFuelClassesMod , only : nfc_fates => num_fuel_classes use FatesConstantsMod , only : n_landuse_cats use FatesLitterMod , only : ncwd_fates => ncwd use FatesInterfaceTypesMod , only : numpft_fates => numpft @@ -1933,7 +1933,7 @@ subroutine htape_create (t, histrest) call ncd_defdim(lnfid, 'fates_levcacls',nlevcoage, dimid) call ncd_defdim(lnfid, 'fates_levpft', numpft_fates, dimid) call ncd_defdim(lnfid, 'fates_levage', nlevage_fates, dimid) - call ncd_defdim(lnfid, 'fates_levfuel', nfsc_fates, dimid) + call ncd_defdim(lnfid, 'fates_levfuel', nfc_fates, dimid) call ncd_defdim(lnfid, 'fates_levcwdsc', ncwd_fates, dimid) call ncd_defdim(lnfid, 'fates_levscpf', nlevsclass_fates*numpft_fates, dimid) call ncd_defdim(lnfid, 'fates_levcapf', nlevcoage*numpft_fates, dimid) @@ -1951,7 +1951,7 @@ subroutine htape_create (t, histrest) call ncd_defdim(lnfid, 'fates_levelpft', nelements_fates * numpft_fates, dimid) call ncd_defdim(lnfid, 'fates_levelcwd', nelements_fates * ncwd_fates, dimid) call ncd_defdim(lnfid, 'fates_levelage', nelements_fates * nlevage_fates, dimid) - call ncd_defdim(lnfid, 'fates_levagefuel', nlevage_fates * nfsc_fates, dimid) + call ncd_defdim(lnfid, 'fates_levagefuel', nlevage_fates * nfc_fates, dimid) call ncd_defdim(lnfid, 'fates_levlanduse', n_landuse_cats, dimid) call ncd_defdim(lnfid, 'fates_levlulu', n_landuse_cats * n_landuse_cats, dimid) end if @@ -4796,7 +4796,7 @@ subroutine hist_addfld2d (fname, type2d, units, avgflag, long_name, type1d_out, case ('fates_levelage') num2d = nelements_fates*nlevage_fates case ('fates_levagefuel') - num2d = nlevage_fates*nfsc_fates + num2d = nlevage_fates*nfc_fates case('cft') if (cft_size > 0) then num2d = cft_size @@ -4842,7 +4842,7 @@ subroutine hist_addfld2d (fname, type2d, units, avgflag, long_name, type1d_out, case ('fates_levage') num2d = nlevage_fates case ('fates_levfuel') - num2d = nfsc_fates + num2d = nfc_fates case ('fates_levcwdsc') num2d = ncwd_fates case ('fates_levscpf') diff --git a/components/elm/src/main/pftvarcon.F90 b/components/elm/src/main/pftvarcon.F90 index fc5bfd54e7d3..66c2adda90f8 100644 --- a/components/elm/src/main/pftvarcon.F90 +++ b/components/elm/src/main/pftvarcon.F90 @@ -29,7 +29,7 @@ module pftvarcon ! ! Vegetation type constants ! - integer :: noveg !value for not vegetated + integer :: noveg !value for not vegetated integer :: ndllf_evr_tmp_tree !value for Needleleaf evergreen temperate tree integer :: ndllf_evr_brl_tree !value for Needleleaf evergreen boreal tree integer :: ndllf_dcd_brl_tree !value for Needleleaf deciduous boreal tree @@ -76,7 +76,7 @@ module pftvarcon integer :: nrtubers !value for root tubers, rain fed (rf) integer :: nrtubersirrig !value for root tubers, irrigated (ir) integer :: nsugarcane !value for sugarcane, rain fed (rf) - integer :: nsugarcaneirrig !value for sugarcane, irrigated (ir) + integer :: nsugarcaneirrig !value for sugarcane, irrigated (ir) integer :: nmiscanthus !value for miscanthus, rain fed (rf) integer :: nmiscanthusirrig !value for miscanthus, irrigated (ir) integer :: nswitchgrass !value for switchgrass, rain fed (rf) @@ -84,8 +84,8 @@ module pftvarcon integer :: npoplar !value for poplar, rain fed (rf) integer :: npoplarirrig !value for poplar, irrigated (ir) integer :: nwillow !value for willow, rain fed (rf) - integer :: nwillowirrig !value for willow, irrigated (ir) - + integer :: nwillowirrig !value for willow, irrigated (ir) + ! Number of crop functional types actually used in the model. This includes each CFT for ! which is_pft_known_to_model is true. Note that this includes irrigated crops even if ! irrigation is turned off in this run: it just excludes crop types that aren't handled @@ -171,7 +171,7 @@ module pftvarcon real(r8), allocatable :: minplanttemp(:) !mininum planting temperature used in Phenology (K) real(r8), allocatable :: senestemp(:) !senescence temperature for perennial crops used in Phenology (K) real(r8), allocatable :: min_days_senes(:) !minimum leaf age to allow for leaf senescence - real(r8), allocatable :: froot_leaf(:) !allocation parameter: new fine root C per new leaf C (gC/gC) + real(r8), allocatable :: froot_leaf(:) !allocation parameter: new fine root C per new leaf C (gC/gC) real(r8), allocatable :: stem_leaf(:) !allocation parameter: new stem c per new leaf C (gC/gC) real(r8), allocatable :: croot_stem(:) !allocation parameter: new coarse root C per new stem C (gC/gC) real(r8), allocatable :: flivewd(:) !allocation parameter: fraction of new wood that is live (phloem and ray parenchyma) (no units) @@ -215,7 +215,7 @@ module pftvarcon real(r8), allocatable :: fyield(:) !fraction of grain that is actually harvested real(r8), allocatable :: root_dmx(:) !maximum root depth - integer, parameter :: pftname_len = 40 ! max length of pftname + integer, parameter :: pftname_len = 40 ! max length of pftname character(len=:), allocatable :: pftname(:) !PFT description real(r8), parameter :: reinickerp = 1.6_r8 !parameter in allometric equation @@ -276,7 +276,7 @@ module pftvarcon real(r8), allocatable :: deadwdcp_obs_flex(:,:) !upper and lower range of dead wood (xylem and heartwood) C:P (gC/gP) ! Photosynthesis parameters real(r8), allocatable :: fnr(:) !fraction of nitrogen in RuBisCO - real(r8), allocatable :: act25(:) + real(r8), allocatable :: act25(:) real(r8), allocatable :: kcha(:) !Activation energy for kc real(r8), allocatable :: koha(:) !Activation energy for ko real(r8), allocatable :: cpha(:) !Activation energy for cp @@ -293,7 +293,7 @@ module pftvarcon real(r8), allocatable :: theta_cj(:) ! real(r8), allocatable :: bbbopt(:) !Ball-Berry stomatal conductance intercept real(r8), allocatable :: mbbopt(:) !Ball-Berry stomatal conductance slope - real(r8), allocatable :: nstor(:) !Nitrogen storage pool timescale + real(r8), allocatable :: nstor(:) !Nitrogen storage pool timescale real(r8), allocatable :: br_xr(:) !Base rate for excess respiration real(r8) :: tc_stress !Critial temperature for moisture stress real(r8), allocatable :: vcmax_np1(:) !vcmax~np relationship coefficient @@ -312,6 +312,13 @@ module pftvarcon real(r8), allocatable :: gcbr_p(:) !effectiveness of roots in reducing rainfall-driven erosion real(r8), allocatable :: gcbr_q(:) !effectiveness of roots in reducing runoff-driven erosion + ! NGEE Arctic snow-vegetation interactions + real(r8), allocatable :: bendresist(:) ! vegetation resistance to bending under snow loading, 0 to 1 (e.g., Liston and Hiemstra 2011; Sturm et al. 2005) + real(r8), allocatable :: vegshape(:) ! shape parameter to modify shrub burial by snow (1 = parabolic, 2 = hemispheric) + real(r8), allocatable :: stocking(:) ! stocking density for pft (stems / hectare) + real(r8), allocatable :: taper(:) ! ratio of height:radius_breast_height (woody vegetation allometry) + logical :: taper_defaults ! set flag to use taper defaults if not on params file (necessary as import and set values are in different places) + ! new pft properties, together with woody, crop, percrop, evergreen, stress_decid, season_decid, defined above, ! are introduced to define vegetation properties. This will be well defineing a pft so that no indices needed for codes. real(r8), allocatable :: climatezone(:) ! distributed climate zone (0 = any zone, 1 = tropical, 2 = temperate, 3 = boreal, 4 = arctic) @@ -379,7 +386,7 @@ subroutine pftconrd ! and finally crops, ending with soybean ! DO NOT CHANGE THE ORDER -- WITHOUT MODIFYING OTHER PARTS OF THE CODE WHERE THE ORDER MATTERS! ! - character(len=pftname_len) :: expected_pftnames(0:mxpft) + character(len=pftname_len) :: expected_pftnames(0:mxpft) !----------------------------------------------------------------------- expected_pftnames( 0) = 'not_vegetated ' @@ -452,95 +459,95 @@ subroutine pftconrd ! but cannot be less, i.e. there cannot be more pfts in the surface file than there are set of pft parameters, if running the non-crop version of model. if (npft mxpft_nc) EXIT ! exit the do loop - + ! (FATES-INTERF) Later, depending on how the team plans to structure the crop model ! or other modules that co-exist while FATES is on, we may want to preserve these pft definitions ! on non-fates columns. For now, they are incompatible, and this check is warranted (rgk 04-2017) @@ -1507,6 +1542,17 @@ subroutine pftconrd end do end if + ! reassign taper values for shrubs - RPF + if (taper_defaults) then + do i = 0, npft-1 + if (woody(i) == 2._r8) then + taper(i) = 10._r8 ! shrubs + else if (woody(i) == 1._r8) then + taper(i) = 200._r8 + end if + end do + end if + if (masterproc) then write(iulog,*) 'Successfully read PFT physiological data' write(iulog,*) diff --git a/components/elm/src/main/surfrdMod.F90 b/components/elm/src/main/surfrdMod.F90 index a8146e5a0f15..97a4fae62d48 100755 --- a/components/elm/src/main/surfrdMod.F90 +++ b/components/elm/src/main/surfrdMod.F90 @@ -20,11 +20,6 @@ module surfrdMod use ncdio_pio , only : ncd_io, check_var, ncd_inqfdims, check_dim, ncd_inqdid, ncd_inqdlen use pio -#ifdef HAVE_MOAB - use mct_mod , only : mct_gsMap - use decompMod , only : get_elmlevel_gsmap - ! use spmdMod , only : iam ! rank on the land communicator -#endif use spmdMod use topounit_varcon , only : max_topounits, has_topounit @@ -184,11 +179,6 @@ subroutine surfrd_get_grid(begg, endg, ldomain, filename, glcfilename) ! pflotran:beg----------------------------- integer :: j, np, nv -#ifdef HAVE_MOAB - type(mct_gsMap), pointer :: gsMap - integer :: i, iv , iseg, ig, local ! ni, nj, nv, nseg, global ig - -#endif ! pflotran:end----------------------------- character(len=32) :: subname = 'surfrd_get_grid' ! subroutine name @@ -258,59 +248,6 @@ subroutine surfrd_get_grid(begg, endg, ldomain, filename, glcfilename) end if ! pflotran:end----------------------------------------------- - -#ifdef HAVE_MOAB - ! read xv and yv for MOAB to learn mesh verticies - if (ldomain%nv>=3 ) then - call get_elmlevel_gsmap (grlnd, gsMap) - allocate(rdata3d(nv,ni,nj)) ! transpose from c, as this is fortran - vname = 'xv' - ! this should be improved in a distributed read, that does not use full grid ni * nj * nv 720*360*4*8 ~ 8Mb - call ncd_io(ncid=ncid, varname=trim(vname), data=rdata3d, flag='read', readvar=readvar) - if (.not. readvar) call endrun( msg=trim(subname)//' ERROR: xv NOT on file'//errMsg(__FILE__, __LINE__)) - ! fill up the ldomain%mblonv(begg:endg, 1:nv) array - local = begg - do iseg = 1, gsMap%ngseg - if (gsMap%pe_loc(iseg) .eq. iam) then - do ig = gsMap%start(iseg), gsMap%start(iseg) + gsMap%length(iseg) - 1 - j = (ig-1)/ni + 1 - i = ig - ni*(j-1) - do iv = 1, nv - if (local .le. endg) then - ldomain%mblonv(local, iv ) = rdata3d(iv, i, j) - else - write (iulog, *), 'OVERFLOW', iseg, gsMap%pe_loc(iseg), gsMap%start(iseg), gsMap%length(iseg), local - endif - enddo - local = local + 1 - enddo - endif - enddo - ! repeat for mblatv - vname = 'yv' - call ncd_io(ncid=ncid, varname=trim(vname), data=rdata3d, flag='read', readvar=readvar) - if (.not. readvar) call endrun( msg=trim(subname)//' ERROR: yv NOT on file'//errMsg(__FILE__, __LINE__)) - ! fill up the ldomain%lonv(begg:endg, 1:nv) array - local = begg - do iseg = 1, gsMap%ngseg - if (gsMap%pe_loc(iseg) .eq. iam) then - do ig = gsMap%start(iseg), gsMap%start(iseg) + gsMap%length(iseg) - 1 - j = (ig-1)/ni + 1 - i = ig - ni*(j-1) - do iv = 1, nv - if (local .le. endg) then - ldomain%mblatv(local, iv ) = rdata3d(iv, i, j) - endif - enddo - local = local + 1 - enddo - endif - enddo - ! deallocate what is not needed anymore (for half degree land model, ~8Mb) - deallocate(rdata3d) - - end if -#endif else call ncd_io(ncid=ncid, varname= 'AREA', flag='read', data=ldomain%area, & dim1name=grlnd, readvar=readvar) diff --git a/components/elm/src/utils/domainMod.F90 b/components/elm/src/utils/domainMod.F90 index 5ef3ae611cf6..2c7771179d21 100755 --- a/components/elm/src/utils/domainMod.F90 +++ b/components/elm/src/utils/domainMod.F90 @@ -52,10 +52,6 @@ module domainMod integer :: nv ! number of vertices real(r8),pointer :: latv(:,:) ! latitude of grid cell's vertices (deg) real(r8),pointer :: lonv(:,:) ! longitude of grid cell's vertices (deg) -#ifdef HAVE_MOAB - real(r8),pointer :: mblatv(:,:) ! latitude of grid cell's vertices (deg) for MOAB - real(r8),pointer :: mblonv(:,:) ! longitude of grid cell's vertices (deg) for MOAB -#endif real(r8) :: lon0 ! the origin lon/lat (Most western/southern corner, if not globally covered grids; OR -180W(360E)/-90N) real(r8) :: lat0 ! the origin lon/lat (Most western/southern corner, if not globally covered grids; OR -180W(360E)/-90N) @@ -154,22 +150,6 @@ subroutine domain_init(domain,isgrid2d,ni,nj,nbeg,nend,elmlevel) endif end if ! pflotran:end----------------------------------------------------- -#ifdef HAVE_MOAB - if (domain%nv > 0 .and. domain%nv /= huge(1)) then - if(.not.associated(domain%mblonv)) then - allocate(domain%mblonv(nb:ne, 1:domain%nv), stat=ier) - if (ier /= 0) & - call shr_sys_abort('domain_init ERROR: allocate mblonv ') - domain%mblonv = nan - endif - if(.not.associated(domain%mblatv)) then - allocate(domain%mblatv(nb:ne, 1:domain%nv)) - if (ier /= 0) & - call shr_sys_abort('domain_init ERROR: allocate mblatv ') - domain%mblatv = nan - endif - end if -#endif if (present(elmlevel)) then domain%elmlevel = elmlevel @@ -265,23 +245,6 @@ subroutine domain_clean(domain) endif endif ! pflotran:beg----------------------------------------------------- -#ifdef HAVE_MOAB - if (domain%nv > 0 .and. domain%nv /= huge(1)) then - if (associated(domain%mblonv)) then - deallocate(domain%mblonv, stat=ier) - if (ier /= 0) & - call shr_sys_abort('domain_clean ERROR: deallocate mblonv ') - nullify(domain%mblonv) - endif - - if (associated(domain%mblatv)) then - deallocate(domain%mblatv, stat=ier) - if (ier /= 0) & - call shr_sys_abort('domain_clean ERROR: deallocate mblatv ') - nullify(domain%mblatv) - endif - endif -#endif else if (masterproc) then diff --git a/components/elm/tools/interpinic/src/fmain.F90 b/components/elm/tools/interpinic/src/fmain.F90 index 0f55c12c67c9..a67e6e2627fb 100644 --- a/components/elm/tools/interpinic/src/fmain.F90 +++ b/components/elm/tools/interpinic/src/fmain.F90 @@ -14,7 +14,7 @@ program fmain character(len= 256) :: arg integer :: n !index integer :: nargs !number of arguments - integer, external :: iargc !number of arguments function + integer :: iargc !number of arguments function character(len=256) :: finidati !input initial dataset to read character(len=256) :: finidato !output initial dataset to create character(len=256) :: cmdline !input command line diff --git a/components/elm/tools/interpinic/src/interpinic.F90 b/components/elm/tools/interpinic/src/interpinic.F90 index b75afd3a411d..3d890a14b895 100644 --- a/components/elm/tools/interpinic/src/interpinic.F90 +++ b/components/elm/tools/interpinic/src/interpinic.F90 @@ -146,9 +146,19 @@ subroutine interp_filei (fin, fout, cmdline) call check_ret (nf90_open(fin, NF90_NOWRITE, ncidi )) call check_ret (nf90_open(fout, NF90_NOWRITE, ncido )) call check_ret (nf_inq_format( ncido, ncformat )) - if ( ncformat /= NF_FORMAT_64BIT )then - write (6,*) 'error: output file is NOT in NetCDF large-file format!' - stop + + ! Allow any format for output dataset + + if ( ncformat == NF_FORMAT_CLASSIC )then + write (6,*) 'info: output file is NF_FORMAT_CLASSIC' + else if ( ncformat == NF_FORMAT_64BIT_OFFSET )then + write (6,*) 'info: output file is NF_FORMAT_64BIT_OFFSET' + else if ( ncformat == NF_FORMAT_64BIT_DATA )then + write (6,*) 'info: output file is NF_FORMAT_64BIT_DATA' + else if ( ncformat == NF_FORMAT_NETCDF4 )then + write (6,*) 'info: output file is NF_FORMAT_NETCDF4' + else if ( ncformat == NF_FORMAT_NETCDF4_CLASSIC )then + write (6,*) 'info: output file is NF_FORMAT_NETCDF4_CLASSIC' end if call check_ret (nf90_inq_dimid(ncidi, "column", dimidcols )) @@ -214,12 +224,25 @@ subroutine interp_filei (fin, fout, cmdline) ret = nf90_inq_dimid(ncidi, "month", dimidmon) if (ret == NF90_NOERR) then call check_ret (nf90_inquire_dimension(ncidi, dimidmon, len=nlevmon)) - call check_ret (nf90_inq_dimid(ncido, "month", dimid )) - call check_ret (nf90_inquire_dimension(ncido, dimid, len=dimlen)) - if (dimlen/=nlevmon) then - write (6,*) 'error: input and output nlevmon values disagree' - write (6,*) 'input nlevmon = ',nlevmon,' output nlevmon = ',dimlen - stop + + ! Many restart files have "month" dimension in input dataset + ! It is only necessary that the output dataset contains "month" dimension + ! when a variable in the input dataset contains the "month" dimension + ! Otherwise, the "month" dimension will never be used + ! Warn rather than die when input has "month" and output does not + + ret = nf90_inq_dimid(ncido, "month", dimid ) + if ( ret == nf_ebaddim ) then + write (6,*) 'warning: input has "month" dimension and output does not' + write (6,*) 'warning: interpolation will fail if any input variable uses "month" dimension' + write (6,*) 'chill: many times the "month" dimension is superfluous so this might work...' + else + call check_ret (nf90_inquire_dimension(ncido, dimid, len=dimlen)) + if (dimlen/=nlevmon) then + write (6,*) 'error: input and output nlevmon values disagree' + write (6,*) 'input nlevmon = ',nlevmon,' output nlevmon = ',dimlen + stop + end if end if else write (6,*) 'month dimension does NOT exist on the input dataset' @@ -321,7 +344,9 @@ subroutine interp_filei (fin, fout, cmdline) ! OK now, open the output file for writing ! call check_ret(nf90_close( ncido)) - call check_ret (nf90_open(fout, ior(NF90_WRITE, NF_64BIT_OFFSET), ncido )) + + ! Allow any format for output dataset + call check_ret (nf90_open(fout, NF90_WRITE, ncido )) call addglobal (ncido, cmdline) @@ -1503,8 +1528,7 @@ subroutine addglobal (ncid, cmdline) character(len=10) :: time character(len= 5) :: zone character(len=18) :: datetime - character(len=256):: version = & - "$HeadURL: https://svn-ccsm-models.cgd.ucar.edu/clm2/trunk_tags/clm4_5_1_r085/models/lnd/clm/tools/clm4_5/interpinic/src/interpinic.F90 $" + character(len=256):: version = "" character(len=256) :: revision_id = "$Id: interpinic.F90 54953 2013-11-06 16:29:45Z sacks $" character(len=16) :: logname character(len=16) :: hostname diff --git a/components/elm/tools/interpinic/src/shr_infnan_mod.F90 b/components/elm/tools/interpinic/src/shr_infnan_mod.F90 index 638cad84d20e..31ffb1dff329 100644 --- a/components/elm/tools/interpinic/src/shr_infnan_mod.F90 +++ b/components/elm/tools/interpinic/src/shr_infnan_mod.F90 @@ -2,11 +2,11 @@ module shr_infnan_mod -!! Inf_NaN_Detection module +!! Inf_NaN_Detection module !! Copyright(c) 2003, Lahey Computer Systems, Inc. -!! Copies of this source code, or standalone compiled files +!! Copies of this source code, or standalone compiled files !! derived from this source may not be sold without permission -!! from Lahey Computers Systems. All or part of this module may be +!! from Lahey Computers Systems. All or part of this module may be !! freely incorporated into executable programs which are offered !! for sale. Otherwise, distribution of all or part of this file is !! permitted, provided this copyright notice and header are included. @@ -22,12 +22,12 @@ module shr_infnan_mod !! isneginf(x) - test for a negative "infinite" value !! !! Each function accepts a single or double precision real argument, and -!! returns a true or false value to indicate the presence of the value +!! returns a true or false value to indicate the presence of the value !! being tested for. If the argument is array valued, the function returns !! a conformable logical array, suitable for use with the ANY function, or !! as a logical mask. !! -!! Each function operates by transferring the bit pattern from a real +!! Each function operates by transferring the bit pattern from a real !! variable to an integer container. Unless testing for + or - infinity, !! the sign bit is cleared to zero. The value is exclusive ORed with !! the value being tested for. The integer result of the IEOR function is @@ -48,14 +48,14 @@ module shr_infnan_mod integer, parameter :: Double = selected_int_kind(precision(1.0_r8)) ! Single precision IEEE values - integer(Single), parameter :: sNaN = Z"7FC00000" - integer(Single), parameter :: sPosInf = Z"7F800000" - integer(Single), parameter :: sNegInf = Z"FF800000" + integer(Single), parameter :: sNaN = int(Z"7FC00000") + integer(Single), parameter :: sPosInf = int(Z"7F800000") + integer(Single), parameter :: sNegInf = int(Z"FF800000") ! Double precision IEEE values - integer(Double), parameter :: dNaN = Z"7FF8000000000000" - integer(Double), parameter :: dPosInf = Z"7FF0000000000000" - integer(Double), parameter :: dNegInf = Z"FFF0000000000000" + integer(Double), parameter :: dNaN = int(Z"7FF8000000000000") + integer(Double), parameter :: dPosInf = int(Z"7FF0000000000000") + integer(Double), parameter :: dNegInf = int(Z"FFF0000000000000") ! Locatation of single and double precision sign bit (Intel) ! Subtract one because bit numbering starts at zero @@ -84,22 +84,22 @@ module shr_infnan_mod module procedure sisnan module procedure disnan #endif - end interface + end interface interface shr_infnan_isinf module procedure sisinf module procedure disinf - end interface - + end interface + interface shr_infnan_isposinf module procedure sisposinf module procedure disposinf - end interface - + end interface + interface shr_infnan_isneginf module procedure sisneginf module procedure disneginf - end interface + end interface integer :: shr_sisnan @@ -107,7 +107,7 @@ module shr_infnan_mod integer :: shr_disnan external :: shr_disnan -contains +contains ! ! If FORTRAN intrinsic's exist use them @@ -134,7 +134,7 @@ elemental function sisnan(x) result(res) res = isnan(x) #endif - end function + end function ! Double precision test for NaN elemental function disnan(d) result(res) @@ -156,7 +156,7 @@ elemental function disnan(d) result(res) res = isnan(d) #endif - end function + end function ! ! Otherwise link to a C function call that either uses the C90 isnan function or a x != x check @@ -176,13 +176,13 @@ function c_sisnan_1D(x) result(res) real(r4), intent(in) :: x(:) logical :: res(size(x)) - integer :: i + integer :: i do i = 1, size(x) res(i) = (shr_sisnan(x(i)) /= 0) end do end function c_sisnan_1D - + function c_sisnan_2D(x) result(res) real(r4), intent(in) :: x(:,:) logical :: res(size(x,1),size(x,2)) @@ -195,7 +195,7 @@ function c_sisnan_2D(x) result(res) end do end do end function c_sisnan_2D - + function c_sisnan_3D(x) result(res) real(r4), intent(in) :: x(:,:,:) logical :: res(size(x,1),size(x,2),size(x,3)) @@ -210,7 +210,7 @@ function c_sisnan_3D(x) result(res) end do end do end function c_sisnan_3D - + function c_sisnan_4D(x) result(res) real(r4), intent(in) :: x(:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4)) @@ -227,7 +227,7 @@ function c_sisnan_4D(x) result(res) end do end do end function c_sisnan_4D - + function c_sisnan_5D(x) result(res) real(r4), intent(in) :: x(:,:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4),size(x,5)) @@ -246,7 +246,7 @@ function c_sisnan_5D(x) result(res) end do end do end function c_sisnan_5D - + function c_sisnan_6D(x) result(res) real(r4), intent(in) :: x(:,:,:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4),size(x,5),size(x,6)) @@ -267,7 +267,7 @@ function c_sisnan_6D(x) result(res) end do end do end function c_sisnan_6D - + function c_sisnan_7D(x) result(res) real(r4), intent(in) :: x(:,:,:,:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4),size(x,5),size(x,6),size(x,7)) @@ -290,7 +290,7 @@ function c_sisnan_7D(x) result(res) end do end do end function c_sisnan_7D - + function c_disnan_scalar(x) result(res) real(r8), intent(in) :: x logical :: res @@ -302,13 +302,13 @@ function c_disnan_1D(x) result(res) real(r8), intent(in) :: x(:) logical :: res(size(x)) - integer :: i + integer :: i do i = 1, size(x) res(i) = (shr_disnan(x(i)) /= 0) end do end function c_disnan_1D - + function c_disnan_2D(x) result(res) real(r8), intent(in) :: x(:,:) logical :: res(size(x,1),size(x,2)) @@ -321,7 +321,7 @@ function c_disnan_2D(x) result(res) end do end do end function c_disnan_2D - + function c_disnan_3D(x) result(res) real(r8), intent(in) :: x(:,:,:) logical :: res(size(x,1),size(x,2),size(x,3)) @@ -336,7 +336,7 @@ function c_disnan_3D(x) result(res) end do end do end function c_disnan_3D - + function c_disnan_4D(x) result(res) real(r8), intent(in) :: x(:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4)) @@ -353,7 +353,7 @@ function c_disnan_4D(x) result(res) end do end do end function c_disnan_4D - + function c_disnan_5D(x) result(res) real(r8), intent(in) :: x(:,:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4),size(x,5)) @@ -372,7 +372,7 @@ function c_disnan_5D(x) result(res) end do end do end function c_disnan_5D - + function c_disnan_6D(x) result(res) real(r8), intent(in) :: x(:,:,:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4),size(x,5),size(x,6)) @@ -393,7 +393,7 @@ function c_disnan_6D(x) result(res) end do end do end function c_disnan_6D - + function c_disnan_7D(x) result(res) real(r8), intent(in) :: x(:,:,:,:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4),size(x,5),size(x,6),size(x,7)) @@ -418,48 +418,48 @@ function c_disnan_7D(x) result(res) end function c_disnan_7D #endif - + ! Single precision test for Inf elemental function sisinf(x) result(res) real(r4), intent(in) :: x logical :: res res = ieor(ibclr(transfer(x,sPosInf),SPSB), sPosInf) == 0 - end function + end function ! Double precision test for Inf elemental function disinf(d) result(res) real(r8), intent(in) :: d logical :: res res = ieor(ibclr(transfer(d,dPosInf),DPSB), dPosInf) == 0 - end function - + end function + ! Single precision test for +Inf elemental function sisposinf(x) result(res) real(r4), intent(in) :: x logical :: res res = ieor(transfer(x,sPosInf), sPosInf) == 0 - end function + end function ! Double precision test for +Inf elemental function disposinf(d) result(res) real(r8), intent(in) :: d logical :: res res = ieor(transfer(d,dPosInf), dPosInf) == 0 - end function - + end function + ! Single precision test for -Inf elemental function sisneginf(x) result(res) real(r4), intent(in) :: x logical :: res res = ieor(transfer(x,sNegInf), sNegInf) == 0 - end function + end function ! Double precision test for -Inf elemental function disneginf(d) result(res) real(r8), intent(in) :: d logical :: res res = ieor(transfer(d,dNegInf), dNegInf) == 0 - end function + end function end module shr_infnan_mod diff --git a/components/homme/CMakeLists.txt b/components/homme/CMakeLists.txt index 6fe81180ab54..4bbfb3d73acf 100644 --- a/components/homme/CMakeLists.txt +++ b/components/homme/CMakeLists.txt @@ -256,9 +256,22 @@ elseif (NOT TARGET csm_share) message (FATAL_ERROR "Aborting.") endif() -OPTION(HOMME_USE_MKL "Whether to use Intel's MKL instead of blas/lapack" FALSE) +OPTION(HOMME_USE_MKL "Whether to use Intel's MKL/oneMKL instead of blas/lapack" FALSE) IF(HOMME_USE_MKL) - MESSAGE(STATUS "HOMME_USE_MKL is ON. The flag -mkl will be added to each executable/library.") + IF(DEFINED ENV{MKLROOT}) + SET(MKL_ROOT_PATH "$ENV{MKLROOT}") + IF(MKL_ROOT_PATH MATCHES "oneapi/mkl") + SET(MKL_TYPE "oneMKL") + MESSAGE(STATUS "Detected oneMKL based on MKLROOT: ${MKL_ROOT_PATH}") + FIND_PACKAGE(MKL REQUIRED $ENV{MKLROOT}/lib/cmake/mkl) + ELSE() + SET(MKL_TYPE "Intel MKL") + MESSAGE(STATUS "Detected standalone Intel MKL based on MKLROOT: ${MKL_ROOT_PATH}") + ENDIF() + ELSE() + MESSAGE(FATAL_ERROR "MKLROOT environment variable is not set. Please set it to your MKL installation path.") + ENDIF() + MESSAGE(STATUS "HOMME_USE_MKL is ON. The flag -mkl/-qmkl will be added to each executable/library.") ELSE() OPTION(HOMME_FIND_BLASLAPACK "Whether to use system blas/lapack" FALSE) MESSAGE(STATUS "HOMME_FIND_BLASLAPACK=${HOMME_FIND_BLASLAPACK}") @@ -308,7 +321,10 @@ IF (HOMME_USE_KOKKOS) IF (CUDA_BUILD OR HIP_BUILD OR SYCL_BUILD) SET (DEFAULT_VECTOR_SIZE 1) SET (HOMMEXX_ENABLE_GPU TRUE) - SET (HOMMEXX_ENABLE_GPU_F90 TRUE) + SET (HOMMEXX_ENABLE_GPU_F90 TRUE) + IF (SYCL_BUILD) + SET (DISABLE_TIMERS_IN_FIRST_STEP TRUE) + ENDIF() ELSE () SET (DEFAULT_VECTOR_SIZE 8) ENDIF() diff --git a/components/homme/cmake/HommeMacros.cmake b/components/homme/cmake/HommeMacros.cmake index b553a8668eba..29a3c64bba9b 100644 --- a/components/homme/cmake/HommeMacros.cmake +++ b/components/homme/cmake/HommeMacros.cmake @@ -112,6 +112,13 @@ macro(createTestExec execName execType macroNP macroNC ADD_DEFINITIONS(-DHAVE_CONFIG_H) ADD_EXECUTABLE(${execName} ${EXEC_SOURCES}) + # For SYCL builds it is suggested to use CXX linker with `-fortlib` + # for mixed-language setups + IF(SYCL_BUILD) + SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE CXX) + ELSE() + SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE Fortran) + ENDIF() IF(BUILD_HOMME_WITHOUT_PIOLIBRARY) TARGET_COMPILE_DEFINITIONS(${execName} PUBLIC HOMME_WITHOUT_PIOLIBRARY) ENDIF() @@ -155,17 +162,20 @@ macro(createTestExec execName execType macroNP macroNC ENDIF () IF (HOMME_USE_KOKKOS) - target_link_libraries(${execName} Kokkos::kokkos) + TARGET_LINK_LIBRARIES(${execName} Kokkos::kokkos) ENDIF () # Move the module files out of the way so the parallel build # doesn't have a race condition SET_TARGET_PROPERTIES(${execName} - PROPERTIES Fortran_MODULE_DIRECTORY ${EXEC_MODULE_DIR}) + PROPERTIES Fortran_MODULE_DIRECTORY ${EXEC_MODULE_DIR}) IF (HOMME_USE_MKL) - TARGET_COMPILE_OPTIONS(${execName} PUBLIC -mkl) - TARGET_LINK_LIBRARIES(${execName} -mkl) + IF (MKL_TYPE STREQUAL "oneMKL") + TARGET_LINK_LIBRARIES(${execName} -qmkl) + ELSEIF (MKL_TYPE STREQUAL "Intel MKL") + TARGET_LINK_LIBRARIES(${execName} -mkl) + ENDIF () ELSE() IF (NOT HOMME_FIND_BLASLAPACK) TARGET_LINK_LIBRARIES(${execName} lapack blas) @@ -263,8 +273,11 @@ macro(createExecLib libName execType libSrcs inclDirs macroNP ENDIF () IF (HOMME_USE_MKL) - TARGET_COMPILE_OPTIONS(${libName} PUBLIC -mkl) - TARGET_LINK_LIBRARIES(${libName} -mkl) + IF (MKL_TYPE STREQUAL "oneMKL") + TARGET_LINK_LIBRARIES(${libName} -qmkl) + ELSEIF (MKL_TYPE STREQUAL "Intel MKL") + TARGET_LINK_LIBRARIES(${libName} -mkl) + ENDIF () ELSE() IF (NOT HOMME_FIND_BLASLAPACK) TARGET_LINK_LIBRARIES(${libName} lapack blas) diff --git a/components/homme/cmake/machineFiles/aurora-aot.cmake b/components/homme/cmake/machineFiles/aurora-aot.cmake new file mode 100644 index 000000000000..14e6d008da84 --- /dev/null +++ b/components/homme/cmake/machineFiles/aurora-aot.cmake @@ -0,0 +1,55 @@ +#module restore +#module load spack-pe-base cmake +#module list + + +SET (AURORA_MACHINE TRUE CACHE BOOL "") + +SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "") +SET(HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "") + +SET(HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "") + +SET(WITH_PNETCDF FALSE CACHE FILEPATH "") + +SET(USE_QUEUING FALSE CACHE BOOL "") + +#temp hack +SET(HOMME_USE_KOKKOS TRUE CACHE BOOL "") +SET(HOMME_USE_MKL TRUE CACHE BOOL "") + +SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "") +SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "") + +set(Kokkos_ROOT $ENV{KOKKOS_HOME} CACHE STRING "") + +SET(USE_TRILINOS OFF CACHE BOOL "") + +SET(SYCL_BUILD TRUE CACHE BOOL "") +SET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "") + +SET(CMAKE_CXX_STANDARD 17) + +SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") +SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") +SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") + +#AOT flags +SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") +SET(SYCL_LINK_FLAGS "-fsycl-max-parallel-link-jobs=32 -fsycl -fsycl-device-code-split=per_kernel -fsycl-targets=intel_gpu_pvc") + +SET(ADD_Fortran_FLAGS "-fc=ifx -fpscomp logicals -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "") +SET(ADD_C_FLAGS "-O3 -DNDEBUG " CACHE STRING "") + +SET(ADD_CXX_FLAGS "-std=c++17 -fp-model=precise -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "") +SET(ADD_LINKER_FLAGS "-O3 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "") + +set (ENABLE_OPENMP OFF CACHE BOOL "") +set (ENABLE_COLUMN_OPENMP OFF CACHE BOOL "") +set (ENABLE_HORIZ_OPENMP OFF CACHE BOOL "") + +set (HOMME_TESTING_PROFILE "dev" CACHE STRING "") + +set (USE_NUM_PROCS 12 CACHE STRING "") + +SET (USE_MPI_OPTIONS "--pmi=pmix --cpu-bind list:0-7,104-111:8-15,112-119:16-23,120-127:24-31,128-135:32-39,136-143:40-47,144-151:52-59,156-163:60-67,164-171:68-75,172-179:76-83,180-187:84-91,188-195:92-99,196-203 gpu_tile_compact.sh" CACHE FILEPATH "") diff --git a/components/homme/cmake/machineFiles/aurora-jit.cmake b/components/homme/cmake/machineFiles/aurora-jit.cmake new file mode 100644 index 000000000000..8c35930fdf66 --- /dev/null +++ b/components/homme/cmake/machineFiles/aurora-jit.cmake @@ -0,0 +1,55 @@ +#module restore +#module load spack-pe-base cmake +#module list + + +SET (AURORA_MACHINE TRUE CACHE BOOL "") + +SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "") +SET(HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "") + +SET(HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "") + +SET(WITH_PNETCDF FALSE CACHE FILEPATH "") + +SET(USE_QUEUING FALSE CACHE BOOL "") + +#temp hack +SET(HOMME_USE_KOKKOS TRUE CACHE BOOL "") +SET(HOMME_USE_MKL TRUE CACHE BOOL "") + +SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "") +SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "") + +set(Kokkos_ROOT $ENV{KOKKOS_HOME} CACHE STRING "") + +SET(USE_TRILINOS OFF CACHE BOOL "") + +SET(SYCL_BUILD TRUE CACHE BOOL "") +SET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "") + +SET(CMAKE_CXX_STANDARD 17) + +SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") +SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") +SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") + +#AOT flags +SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") +SET(SYCL_LINK_FLAGS "-fsycl-max-parallel-link-jobs=32 -fsycl") + +SET(ADD_Fortran_FLAGS "-fc=ifx -fpscomp logicals -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "") +SET(ADD_C_FLAGS "-O3 -DNDEBUG " CACHE STRING "") + +SET(ADD_CXX_FLAGS "-std=c++17 -fp-model=precise -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "") +SET(ADD_LINKER_FLAGS "-O3 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "") + +set (ENABLE_OPENMP OFF CACHE BOOL "") +set (ENABLE_COLUMN_OPENMP OFF CACHE BOOL "") +set (ENABLE_HORIZ_OPENMP OFF CACHE BOOL "") + +set (HOMME_TESTING_PROFILE "dev" CACHE STRING "") + +set (USE_NUM_PROCS 12 CACHE STRING "") + +SET (USE_MPI_OPTIONS "--pmi=pmix --cpu-bind list:0-7,104-111:8-15,112-119:16-23,120-127:24-31,128-135:32-39,136-143:40-47,144-151:52-59,156-163:60-67,164-171:68-75,172-179:76-83,180-187:84-91,188-195:92-99,196-203 gpu_tile_compact.sh" CACHE FILEPATH "") diff --git a/components/homme/cmake/machineFiles/perlmutter-gnu.cmake b/components/homme/cmake/machineFiles/perlmutter-gnu.cmake index a9ad558677ab..a27f83900c1f 100644 --- a/components/homme/cmake/machineFiles/perlmutter-gnu.cmake +++ b/components/homme/cmake/machineFiles/perlmutter-gnu.cmake @@ -13,7 +13,7 @@ SET(HDF5_DIR $ENV{CRAY_HDF5_PARALLEL_PREFIX} CACHE FILEPATH "") SET (NetCDF_C_PATH $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} CACHE FILEPATH "") SET (NetCDF_Fortran_PATH $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} CACHE FILEPATH "") -SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "") +SET(BUILD_HOMME_WITHOUT_PIOLIBRARY FALSE CACHE BOOL "") SET(HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "") @@ -31,6 +31,7 @@ SET(Kokkos_ENABLE_OPENMP OFF CACHE BOOL "") SET(Kokkos_ENABLE_CUDA ON CACHE BOOL "") SET(Kokkos_ENABLE_CUDA_LAMBDA ON CACHE BOOL "") SET(Kokkos_ARCH_AMPERE80 ON CACHE BOOL "") +SET(Kokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC OFF CACHE BOOL "") #SET(Kokkos_ARCH_ZEN2 ON CACHE BOOL "") # works, and perf same if both AMPERE80 and ZEN2 are on #SET(Kokkos_ENABLE_CUDA_UVM ON CACHE BOOL "") SET(Kokkos_ENABLE_EXPLICIT_INSTANTIATION OFF CACHE BOOL "") @@ -42,7 +43,10 @@ SET(Kokkos_ENABLE_EXPLICIT_INSTANTIATION OFF CACHE BOOL "") SET(CMAKE_C_COMPILER "cc" CACHE STRING "") SET(CMAKE_Fortran_COMPILER "ftn" CACHE STRING "") SET(CMAKE_CXX_COMPILER "CC" CACHE STRING "") -# Note: need to set MPICH_CXX env variable and perhaps NVCC_WRAPPER_DEFAULT_COMPILER +# Note: No longer need to set MPICH_CXX env variable and perhaps +# NVCC_WRAPPER_DEFAULT_COMPILER. Ignore the warning about nvcc_wrapper during +# configuration. +SET(CUDA_BUILD TRUE CACHE STRING "") SET(CXXLIB_SUPPORTED_CACHE FALSE CACHE BOOL "") diff --git a/components/homme/cmake/machineFiles/polaris-a100.sh b/components/homme/cmake/machineFiles/polaris-a100.sh new file mode 100644 index 000000000000..2b63c61a55e7 --- /dev/null +++ b/components/homme/cmake/machineFiles/polaris-a100.sh @@ -0,0 +1,74 @@ +#Currently Loaded Modules: +# 1) craype-x86-rome 6) craype/2.7.15 11) cray-libpals/1.1.7 16) nvhpc-mixed/21.9 +# 2) libfabric/1.11.0.4.125 7) cray-dsmml/0.2.2 12) PrgEnv-gnu/8.3.3 17) cudatoolkit-standalone/11.6.2 +# 3) craype-network-ofi 8) cray-pmi/6.1.2 13) gnu-parallel/2021-09-22 18) cmake/3.23.2 +# 4) perftools-base/22.05.0 9) cray-pmi-lib/6.0.17 14) gcc/11.2.0 +# 5) craype-accel-nvidia80 10) cray-pals/1.1.7 15) cray-mpich/8.1.16 + + + +#SET(HOMMEXX_EXEC_SPACE CUDA CACHE STRING "") +#SET(HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "") +#SET(HOMMEXX_CUDA_MAX_WARP_PER_TEAM "16" CACHE STRING "") + +# cray-hdf5-parallel/1.12.0.6 cray-netcdf-hdf5parallel/4.7.4.6 cray-parallel-netcdf/1.12.1.6 +#SET(NETCDF_DIR $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} CACHE FILEPATH "") +#SET(PNETCDF_DIR $ENV{CRAY_PARALLEL_NETCDF_DIR} CACHE FILEPATH "") +#SET(HDF5_DIR $ENV{CRAY_HDF5_PARALLEL_PREFIX} CACHE FILEPATH "") + +#for scorpio +#SET (NetCDF_C_PATH $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} CACHE FILEPATH "") +#SET (NetCDF_Fortran_PATH $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} CACHE FILEPATH "") + +SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "") + +SET(HOMME_FIND_BLASLAPACK FALSE CACHE BOOL "") + +SET(WITH_PNETCDF FALSE CACHE FILEPATH "") + +SET(USE_QUEUING FALSE CACHE BOOL "") + +SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "") + +SET(CUDA_BUILD TRUE CACHE BOOL "") + +#SET(HOMMEXX_BFB_TESTING TRUE CACHE BOOL "") + +SET(USE_TRILINOS OFF CACHE BOOL "") + +SET(Kokkos_ENABLE_OPENMP OFF CACHE BOOL "") +SET(Kokkos_ENABLE_CUDA ON CACHE BOOL "") +SET(Kokkos_ENABLE_CUDA_LAMBDA ON CACHE BOOL "") +SET(Kokkos_ARCH_AMPERE80 ON CACHE BOOL "") +#SET(Kokkos_ARCH_ZEN2 ON CACHE BOOL "") # works, and perf same if both AMPERE80 and ZEN2 are on +#SET(Kokkos_ENABLE_CUDA_UVM ON CACHE BOOL "") +SET(Kokkos_ENABLE_EXPLICIT_INSTANTIATION OFF CACHE BOOL "") +#SET(Kokkos_ENABLE_CUDA_ARCH_LINKING OFF CACHE BOOL "") + +#SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") +#SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") +#SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") +SET(CMAKE_C_COMPILER "cc" CACHE STRING "") +SET(CMAKE_Fortran_COMPILER "ftn" CACHE STRING "") +SET(CMAKE_CXX_COMPILER "CC" CACHE STRING "") + +#SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") +#SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") +#SET(CMAKE_CXX_COMPILER "${CMAKE_CURRENT_SOURCE_DIR}/../../externals/kokkos/bin/nvcc_wrapper" CACHE STRING "") + +# Note: need to set MPICH_CXX env variable and perhaps NVCC_WRAPPER_DEFAULT_COMPILER + +SET(CXXLIB_SUPPORTED_CACHE FALSE CACHE BOOL "") + +SET(ENABLE_OPENMP OFF CACHE BOOL "") +SET(ENABLE_COLUMN_OPENMP OFF CACHE BOOL "") +SET(ENABLE_HORIZ_OPENMP OFF CACHE BOOL "") + +SET(CMAKE_VERBOSE_MAKEFILE ON CACHE BOOL "") + +#SET(HOMME_TESTING_PROFILE "dev" CACHE STRING "") + +SET(USE_NUM_PROCS 4 CACHE STRING "") + +SET(USE_MPIEXEC "srun" CACHE STRING "") +#SET(CPRNC_DIR /global/cfs/cdirs/e3sm/tools/cprnc CACHE FILEPATH "") diff --git a/components/homme/cmake/machineFiles/spot-aot-AB2.cmake b/components/homme/cmake/machineFiles/spot-aot-AB2.cmake new file mode 100644 index 000000000000..23fad2361ccf --- /dev/null +++ b/components/homme/cmake/machineFiles/spot-aot-AB2.cmake @@ -0,0 +1,63 @@ +#module restore +#module load oneapi/eng-compiler/2022.12.30.005 +#module load intel_compute_runtime/release/agama-devel-627 +#module load spack cmake +#module list + +SET (SUNSPOT_MACHINE TRUE CACHE BOOL "") + +SET (HOMMEXX_MPI_ON_DEVICE TRUE CACHE BOOL "") + +#SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "") + +SET(HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "") + +SET(WITH_PNETCDF FALSE CACHE FILEPATH "") + +SET(USE_QUEUING FALSE CACHE BOOL "") + +#temp hack +SET(HOMME_USE_KOKKOS TRUE CACHE BOOL "") + +SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "") +SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "") + +#set(KOKKOS_HOME "/home/onguba/kokkos-build/june22-2024-aot/install" CACHE STRING "") +#set(E3SM_KOKKOS_PATH ${KOKKOS_HOME} CACHE STRING "") + +SET (NetCDF_Fortran_PATH "/lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf" CACHE STRING "") +SET (NetCDF_C_PATH "/lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf" CACHE STRING "") + +SET(USE_TRILINOS OFF CACHE BOOL "") + +SET(SYCL_BUILD TRUE CACHE BOOL "") +SET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "") + +#SET(CMAKE_CXX_STANDARD 17) +SET(CMAKE_CXX_STANDARD 17 CACHE STRING "CXX Standard") + +SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") +SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") +SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") + +SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") +SET(SYCL_LINK_FLAGS "-fsycl-max-parallel-link-jobs=32 -fsycl-link-huge-device-code -fsycl -fsycl-device-code-split=per_kernel -fsycl-targets=spir64_gen -Xsycl-target-backend \"-device 12.60.7\"") + +#-fpscomp does not actually solve the issue with bools in here,another suggestion was -fp-model=precise, not working either +SET(ADD_Fortran_FLAGS " -fc=ifx -fpscomp logicals -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "") +SET(ADD_C_FLAGS "-O3 -DNDEBUG " CACHE STRING "") + +SET(ADD_CXX_FLAGS " -std=c++17 -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "") +SET(ADD_LINKER_FLAGS "-O3 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "") + +set (ENABLE_OPENMP OFF CACHE BOOL "") +set (ENABLE_COLUMN_OPENMP OFF CACHE BOOL "") +set (ENABLE_HORIZ_OPENMP OFF CACHE BOOL "") + +set (HOMME_TESTING_PROFILE "dev" CACHE STRING "") + +set (USE_NUM_PROCS 4 CACHE STRING "") + +SET (USE_MPI_OPTIONS "--bind-to core" CACHE FILEPATH "") + + diff --git a/components/homme/src/preqx/prim_advection_mod.F90 b/components/homme/src/preqx/prim_advection_mod.F90 index 8e7fd8b0cfb7..ecbc745c7304 100644 --- a/components/homme/src/preqx/prim_advection_mod.F90 +++ b/components/homme/src/preqx/prim_advection_mod.F90 @@ -6,14 +6,13 @@ module prim_advection_mod use dimensions_mod, only : nlev, qsize, nelemd use kinds, only : real_kind - use parallel_mod, only : parallel_t + use parallel_mod, only : parallel_t, abortmp use derivative_mod, only : derivative_t use element_mod, only : element_t use hybvcoord_mod, only : hvcoord_t use time_mod, only : TimeLevel_t use hybrid_mod, only : hybrid_t use control_mod, only : transport_alg - use sl_advection, only : prim_advec_tracers_remap_ALE, sl_init1 use prim_advection_base, only: prim_advec_init1_rk2, prim_advec_tracers_remap_rk2,& prim_advec_init2 @@ -35,12 +34,20 @@ subroutine Prim_Advec_Init1(par, elem) type (element_t) :: elem(:) call prim_advec_init1_rk2(par, elem) - call sl_init1(par,elem) - end subroutine Prim_Advec_Init1 + subroutine Prim_Advec_Tracers_observe_velocity(elem, tl, n, nets, nete) + type (element_t) , intent(inout) :: elem(:) + type (TimeLevel_t) , intent(in ) :: tl + integer , intent(in ) :: n + integer , intent(in ) :: nets + integer , intent(in ) :: nete + + ! Do nothing. Only SL transport uses this routine, and it's not supported in + ! preqx. + end subroutine Prim_Advec_Tracers_observe_velocity - subroutine Prim_Advec_Tracers_remap( elem , deriv , hvcoord , hybrid , dt , tl , nets , nete ) + subroutine Prim_Advec_Tracers_remap( elem , deriv , hvcoord , hybrid , dt , tl , nets , nete ) implicit none type (element_t) , intent(inout) :: elem(:) type (derivative_t) , intent(in ) :: deriv @@ -54,8 +61,8 @@ subroutine Prim_Advec_Tracers_remap( elem , deriv , hvcoord , hybrid , dt , tl if (transport_alg == 0) then call Prim_Advec_Tracers_remap_rk2( elem , deriv , hvcoord , hybrid , dt , tl , nets , nete ) - else - call Prim_Advec_Tracers_remap_ALE( elem , deriv , hvcoord , hybrid , dt , tl , nets , nete ) + else + call abortmp('Semi-Lagrangian transport is not supported in preqx.') end if end subroutine Prim_Advec_Tracers_remap diff --git a/components/homme/src/preqx_acc/prim_advection_mod.F90 b/components/homme/src/preqx_acc/prim_advection_mod.F90 index dbf055eb9207..694c8440530c 100644 --- a/components/homme/src/preqx_acc/prim_advection_mod.F90 +++ b/components/homme/src/preqx_acc/prim_advection_mod.F90 @@ -41,6 +41,7 @@ module prim_advection_mod logical, private :: first_time = .true. public :: Prim_Advec_Tracers_remap + public :: Prim_Advec_Tracers_observe_velocity public :: prim_advec_init1 public :: prim_advec_init2 @@ -302,6 +303,17 @@ subroutine prim_advec_init2(elem,hvcoord,hybrid) !$omp barrier end subroutine prim_advec_init2 + subroutine Prim_Advec_Tracers_observe_velocity(elem, tl, n, nets, nete) + type (element_t) , intent(inout) :: elem(:) + type (TimeLevel_t) , intent(in ) :: tl + integer , intent(in ) :: n + integer , intent(in ) :: nets + integer , intent(in ) :: nete + + ! Do nothing. Only SL transport uses this routine, and it's not supported in + ! preqx. + end subroutine Prim_Advec_Tracers_observe_velocity + subroutine advance_hypervis_scalar( elem , hvcoord , hybrid , deriv , nt , nt_qdp , nets , nete , dt2 ) ! hyperviscsoity operator for foward-in-time scheme ! take one timestep of: diff --git a/components/homme/src/preqx_kokkos/CMakeLists.txt b/components/homme/src/preqx_kokkos/CMakeLists.txt index dff42bb97c66..53691c9f2daa 100644 --- a/components/homme/src/preqx_kokkos/CMakeLists.txt +++ b/components/homme/src/preqx_kokkos/CMakeLists.txt @@ -115,7 +115,7 @@ MACRO(PREQX_KOKKOS_SETUP) ${TEST_SRC_DIR}/dcmip12_wrapper.F90 ${TEST_SRC_DIR}/dcmip16_wrapper.F90 ${TEST_SRC_DIR}/dcmip2012_test1_2_3.F90 - ${TEST_SRC_DIR}/dcmip2012_test1_conv.F90 + ${TEST_SRC_DIR}/dcmip2012_test1_conv_mod.F90 ${TEST_SRC_DIR}/dcmip2012_test4.F90 ${TEST_SRC_DIR}/dcmip2012_test5.F90 ${TEST_SRC_DIR}/dcmip2016-baroclinic.F90 diff --git a/components/homme/src/preqx_kokkos/prim_advection_mod.F90 b/components/homme/src/preqx_kokkos/prim_advection_mod.F90 index b3d5595b874a..07895e67e8ea 100644 --- a/components/homme/src/preqx_kokkos/prim_advection_mod.F90 +++ b/components/homme/src/preqx_kokkos/prim_advection_mod.F90 @@ -39,6 +39,16 @@ subroutine Prim_Advec_Init1(par, elem) end subroutine Prim_Advec_Init1 + subroutine Prim_Advec_Tracers_observe_velocity(elem, tl, n, nets, nete) + type (element_t) , intent(inout) :: elem(:) + type (TimeLevel_t) , intent(in ) :: tl + integer , intent(in ) :: n + integer , intent(in ) :: nets + integer , intent(in ) :: nete + + ! Do nothing. Only SL transport uses this routine, and it's not supported in + ! preqx. + end subroutine Prim_Advec_Tracers_observe_velocity subroutine Prim_Advec_Tracers_remap( elem , deriv , hvcoord , hybrid , dt , tl , nets , nete ) implicit none diff --git a/components/homme/src/prim_main.F90 b/components/homme/src/prim_main.F90 index d6901151d365..fcd8a8473023 100644 --- a/components/homme/src/prim_main.F90 +++ b/components/homme/src/prim_main.F90 @@ -68,6 +68,7 @@ end subroutine finalize_kokkos_f90 character (len=20) :: numtrac_char logical :: dir_e ! boolean existence of directory where output netcdf goes + logical :: call_enablef ! ===================================================== ! Begin executable code set distributed memory world... @@ -228,7 +229,20 @@ end subroutine finalize_kokkos_f90 if(par%masterproc) print *,"Entering main timestepping loop" call t_startf('prim_main_loop') + call_enablef = .false. do while(tl%nstep < nEndStep) +#ifdef DISABLE_TIMERS_IN_FIRST_STEP + ! Certain compilers, e.g., for Intel GPU, do just-in-time compilation. Turn + ! off timers in the first step to avoid counting that cost. + if (tl%nstep == 0) then + call t_disablef() + call_enablef = .true. + elseif (call_enablef) then + call t_enablef() + call_enablef = .false. + end if +#endif + #if (defined HORIZ_OPENMP) !$OMP PARALLEL NUM_THREADS(hthreads), DEFAULT(SHARED), PRIVATE(ithr,nets,nete,hybrid) call omp_set_num_threads(vthreads) diff --git a/components/homme/src/share/compose/CMakeLists.txt b/components/homme/src/share/compose/CMakeLists.txt index a052dcc30326..6bec15c08a09 100644 --- a/components/homme/src/share/compose/CMakeLists.txt +++ b/components/homme/src/share/compose/CMakeLists.txt @@ -30,12 +30,14 @@ add_library (${COMPOSE_LIBRARY} compose_slmm_islmpi_pack.cpp compose_slmm_islmpi_q.cpp compose_slmm_islmpi_qextrema.cpp + compose_slmm_islmpi_interpolate.cpp compose_slmm_islmpi_step.cpp compose_cedr_sl_run_global.cpp compose_cedr_sl_run_local.cpp compose_cedr_sl_run_check.cpp compose_cedr_qlt.cpp compose_cedr_caas.cpp + compose_slmm_islmpi_calc_trajectory.cpp cedr_util.cpp cedr_mpi.cpp cedr_local.cpp diff --git a/components/homme/src/share/compose/compose_cedr.cpp b/components/homme/src/share/compose/compose_cedr.cpp index 0aeaebf9487c..9a8648d86da7 100644 --- a/components/homme/src/share/compose/compose_cedr.cpp +++ b/components/homme/src/share/compose/compose_cedr.cpp @@ -419,12 +419,12 @@ struct TreeReducer : }; template -CDR::CDR (Int cdr_alg_, Int ngblcell_, Int nlclcell_, Int nlev_, Int qsize_, - bool use_sgi, bool independent_time_steps, const bool hard_zero_, - const Int* gid_data, const Int* rank_data, +CDR::CDR (Int cdr_alg_, Int ngblcell_, Int nlclcell_, Int nlev_, Int np_, + Int qsize_, bool use_sgi, bool independent_time_steps, + const bool hard_zero_, const Int* gid_data, const Int* rank_data, const cedr::mpi::Parallel::Ptr& p_, Int fcomm) : alg(Alg::convert(cdr_alg_)), - ncell(ngblcell_), nlclcell(nlclcell_), nlev(nlev_), qsize(qsize_), + ncell(ngblcell_), nlclcell(nlclcell_), nlev(nlev_), np(np_), qsize(qsize_), nsublev(Alg::is_suplev(alg) ? nsublev_per_suplev : 1), nsuplev((nlev + nsublev - 1) / nsublev), threed(independent_time_steps), @@ -444,8 +444,9 @@ CDR::CDR (Int cdr_alg_, Int ngblcell_, Int nlclcell_, Int nlev_, Int qsize_, cdr = std::make_shared(p, nleaf, tree, options, threed ? nsuplev : 0); tree = nullptr; } else if (Alg::is_caas(alg)) { - const Int n_accum_in_place = n_id_in_suplev*(cdr_over_super_levels ? - nsuplev : 1); + const Int n_accum_in_place = (n_id_in_suplev* + (Alg::is_point(alg) ? np*np : 1)* + (cdr_over_super_levels ? nsuplev : 1)); typename CAAST::UserAllReducer::Ptr reducer; //todo Measure perf on CPU and GPU of TreeReducer vs // ReproSumReducer. For now, I'll continue to use ReproSumReducer. @@ -458,7 +459,8 @@ CDR::CDR (Int cdr_alg_, Int ngblcell_, Int nlclcell_, Int nlev_, Int qsize_, } else { reducer = std::make_shared >(fcomm, n_accum_in_place); } - const auto caas = std::make_shared(p, nlclcell*n_accum_in_place, reducer); + const auto caas = std::make_shared(p, nlclcell*n_accum_in_place, + reducer); cdr = caas; } else { cedr_throw_if(true, "Invalid semi_lagrange_cdr_alg " << alg); @@ -502,11 +504,13 @@ void set_ie2gci (CDR& q, const Int ie, const Int gci) { q.ie2gci_h[ie] = gci template void init_ie2lci (CDR& q) { + const Int n_in_elem = Alg::is_point(q.alg) ? q.np*q.np : 1; const Int n_id_in_suplev = q.caas_in_suplev ? 1 : q.nsublev; const Int nleaf = n_id_in_suplev* q.ie2gci.size()* - (q.cdr_over_super_levels ? q.nsuplev : 1); + (q.cdr_over_super_levels ? q.nsuplev : 1)* + n_in_elem; q.ie2lci = typename CDR::Idxs("ie2lci", nleaf); q.ie2lci_h = Kokkos::create_mirror_view(q.ie2lci); if (Alg::is_qlt(q.alg)) { @@ -516,9 +520,9 @@ void init_ie2lci (CDR& q) { for (size_t ie = 0; ie < q.ie2gci_h.size(); ++ie) for (Int spli = 0; spli < q.nsuplev; ++spli) for (Int sbli = 0; sbli < n_id_in_suplev; ++sbli) - // local indexing is fastest over the whole column + // Local indexing is fastest over the whole column ... q.ie2lci_h[nlevwrem*ie + n_id_in_suplev*spli + sbli] = - // but global indexing is organized according to the tree + // ... but global indexing is organized according to the tree. qlt->gci2lci(n_id_in_suplev*(q.ncell*spli + q.ie2gci_h[ie]) + sbli); } else { for (size_t ie = 0; ie < q.ie2gci_h.size(); ++ie) @@ -531,16 +535,18 @@ void init_ie2lci (CDR& q) { const auto nlevwrem = q.nsuplev*n_id_in_suplev; for (size_t ie = 0; ie < q.ie2gci_h.size(); ++ie) for (Int spli = 0; spli < q.nsuplev; ++spli) - for (Int sbli = 0; sbli < n_id_in_suplev; ++sbli) { - const Int id = nlevwrem*ie + n_id_in_suplev*spli + sbli; - q.ie2lci_h[id] = id; - } + for (Int sbli = 0; sbli < n_id_in_suplev; ++sbli) + for (Int k = 0; k < n_in_elem; ++k) { + const Int id = nlevwrem*(n_in_elem*ie + k) + n_id_in_suplev*spli + sbli; + q.ie2lci_h[id] = id; + } } else { for (size_t ie = 0; ie < q.ie2gci_h.size(); ++ie) - for (Int sbli = 0; sbli < n_id_in_suplev; ++sbli) { - const Int id = n_id_in_suplev*ie + sbli; - q.ie2lci_h[id] = id; - } + for (Int sbli = 0; sbli < n_id_in_suplev; ++sbli) + for (Int k = 0; k < n_in_elem; ++k) { + const Int id = n_id_in_suplev*(n_in_elem*ie + k) + sbli; + q.ie2lci_h[id] = id; + } } } Kokkos::deep_copy(q.ie2lci, q.ie2lci_h); @@ -625,12 +631,12 @@ extern "C" void cedr_init_impl (const homme::Int fcomm, const homme::Int cdr_alg, const bool use_sgi, const homme::Int* gid_data, const homme::Int* rank_data, const homme::Int gbl_ncell, const homme::Int lcl_ncell, - const homme::Int nlev, const homme::Int qsize, + const homme::Int nlev, const homme::Int np, const homme::Int qsize, const bool independent_time_steps, const bool hard_zero, const homme::Int, const homme::Int) { const auto p = cedr::mpi::make_parallel(MPI_Comm_f2c(fcomm)); g_cdr = std::make_shared >( - cdr_alg, gbl_ncell, lcl_ncell, nlev, qsize, use_sgi, + cdr_alg, gbl_ncell, lcl_ncell, nlev, np, qsize, use_sgi, independent_time_steps, hard_zero, gid_data, rank_data, p, fcomm); } @@ -650,17 +656,7 @@ extern "C" void cedr_set_bufs (homme::Real* sendbuf, homme::Real* recvbuf, extern "C" void cedr_set_null_bufs () { cedr_set_bufs(nullptr, nullptr, 0, 0); } extern "C" void cedr_unittest (const homme::Int fcomm, homme::Int* nerrp) { -#if 0 - auto p = cedr::mpi::make_parallel(MPI_Comm_f2c(fcomm)); - cedr_assert(g_cdr); - cedr_assert(g_cdr->tree); - if (homme::CDR::Alg::is_qlt(g_cdr->alg)) - *nerrp = cedr::qlt::test::test_qlt(p, g_cdr->tree, g_cdr->nsublev*g_cdr->ncell, - 1, false, false, true, false); - else - *nerrp = cedr::caas::test::unittest(p); -#endif - *nerrp += compose::test::cedr_unittest(); + *nerrp = compose::test::cedr_unittest(); } extern "C" void cedr_set_ie2gci (const homme::Int ie, const homme::Int gci) { @@ -715,7 +711,6 @@ extern "C" void cedr_sl_run_global (homme::Real* minq, const homme::Real* maxq, cedr_assert(g_cdr); cedr_assert(g_sl); { homme::Timer timer("h2d"); - //if (g_cdr->p->amroot() && s_h2d) printf("cedr_h2d\n"); homme::cedr_h2d(*g_sl->ta, s_h2d); } homme::sl::run_global(*g_cdr, *g_sl, minq, maxq, nets-1, nete-1); } @@ -730,7 +725,6 @@ extern "C" void cedr_sl_run_local (homme::Real* minq, const homme::Real* maxq, homme::sl::run_local(*g_cdr, *g_sl, minq, maxq, nets-1, nete-1, use_ir, limiter_option); { homme::Timer timer("d2h"); - //if (g_cdr->p->amroot() && s_d2h) printf("cedr_d2h\n"); homme::cedr_d2h(*g_sl->ta, s_d2h); } } diff --git a/components/homme/src/share/compose/compose_cedr_cdr.hpp b/components/homme/src/share/compose/compose_cedr_cdr.hpp index 1c15b364fdfa..fd8e92bc703e 100644 --- a/components/homme/src/share/compose/compose_cedr_cdr.hpp +++ b/components/homme/src/share/compose/compose_cedr_cdr.hpp @@ -8,7 +8,8 @@ namespace homme { struct Alg { - enum Enum { qlt, qlt_super_level, qlt_super_level_local_caas, caas, caas_super_level }; + enum Enum { qlt, qlt_super_level, qlt_super_level_local_caas, caas, + caas_super_level }; static Enum convert (int cdr_alg) { switch (cdr_alg) { case 2: return qlt; @@ -27,6 +28,9 @@ struct Alg { static bool is_caas (Enum e) { return e == caas || e == caas_super_level; } + static bool is_point (Enum e) { + return false; + } static bool is_suplev (Enum e) { return (e == qlt_super_level || e == caas_super_level || e == qlt_super_level_local_caas); @@ -55,7 +59,7 @@ struct CDR { enum { nsublev_per_suplev = 8 }; const Alg::Enum alg; - const Int ncell, nlclcell, nlev, qsize, nsublev, nsuplev; + const Int ncell, nlclcell, nlev, np, qsize, nsublev, nsuplev; const bool threed, cdr_over_super_levels, caas_in_suplev, hard_zero; const cedr::mpi::Parallel::Ptr p; cedr::tree::Node::Ptr tree; // Don't need this except for unit testing. @@ -67,9 +71,10 @@ struct CDR { BoolsH nonneg_h; bool run; // for debugging, it can be useful not to run the CEDR. - CDR(Int cdr_alg_, Int ngblcell_, Int nlclcell_, Int nlev_, Int qsize_, bool use_sgi, - bool independent_time_steps, const bool hard_zero_, const Int* gid_data, - const Int* rank_data, const cedr::mpi::Parallel::Ptr& p_, Int fcomm); + CDR(Int cdr_alg_, Int ngblcell_, Int nlclcell_, Int nlev_, Int np_, Int qsize_, + bool use_sgi, bool independent_time_steps, const bool hard_zero_, + const Int* gid_data, const Int* rank_data, const cedr::mpi::Parallel::Ptr& p_, + Int fcomm); CDR(const CDR&) = delete; CDR& operator=(const CDR&) = delete; diff --git a/components/homme/src/share/compose/compose_homme.cpp b/components/homme/src/share/compose/compose_homme.cpp index 4399e0b5cf9f..938403d096f1 100644 --- a/components/homme/src/share/compose/compose_homme.cpp +++ b/components/homme/src/share/compose/compose_homme.cpp @@ -9,7 +9,6 @@ TracerArrays::TracerArrays (Int nelemd_, Int nlev_, Int np_, Int qsize_, Int pdp(nelemd, np2, nlev), pdp3d(nelemd, np2, nlev, -1, 3), pqdp(nelemd, np2, nlev, qsized, 2), pq(nelemd, np2, nlev, qsized), #if defined COMPOSE_PORT - dep_points("dep_points", nelemd, nlev, np2), q_min("q_min", nelemd, qsize, np2, nlev), q_max("q_max", nelemd, qsize, np2, nlev) #else @@ -31,46 +30,109 @@ void TracerArrays::alloc_if_not () { #endif template -void sl_h2d (TracerArrays& ta, bool transfer, Cartesian3D* dep_points) { +void sl_traj_h2d (TracerArrays& ta, Real* dep_points, Real* vnode, + Real* vdep, Int ndim) { #if defined COMPOSE_PORT +# if defined COMPOSE_HORIZ_OPENMP +# pragma omp master + { +# endif ko::fence(); ta.alloc_if_not(); const Int nelemd = ta.nelemd, qsize = ta.qsize, np2 = ta.np2, nlev = ta.nlev; - const DepPointsH cart_h(reinterpret_cast(dep_points), nelemd, nlev, np2); - const auto dep_points_h = ko::create_mirror_view(ta.dep_points); - for (Int ie = 0; ie < nelemd; ++ie) - for (Int lev = 0; lev < nlev; ++lev) - for (Int k = 0; k < np2; ++k) - for (Int d = 0; d < 3; ++d) - dep_points_h(ie,lev,k,d) = cart_h(ie,lev,k,d); - ko::deep_copy(ta.dep_points, dep_points_h); - if ( ! transfer) return; - const auto qdp_m = ko::create_mirror_view(ta.qdp); - const auto dp_m = ko::create_mirror_view(ta.dp); + const DepPointsH cart_h(dep_points, nelemd, nlev, np2, ndim); + ko::deep_copy(ta.dep_points, cart_h); + if (vnode) { + const DepPointsH h(vnode, nelemd, nlev, np2, ndim); + ko::deep_copy(ta.vnode, h); + } + if (vdep) { + const DepPointsH h(vdep, nelemd, nlev, np2, ndim); + ko::deep_copy(ta.vdep, h); + } +# ifdef COMPOSE_HORIZ_OPENMP + } +# pragma omp barrier +# endif +#endif +} + +template +void sl_traj_d2h (const TracerArrays& ta, Real* dep_points, Real* vnode, + Real* vdep, Int ndim) { +#if defined COMPOSE_PORT +# if defined COMPOSE_HORIZ_OPENMP +# pragma omp master + { +# endif + ko::fence(); const auto q_m = ko::create_mirror_view(ta.q); - for (Int ie = 0; ie < nelemd; ++ie) - for (Int iq = 0; iq < qsize; ++iq) + const Int nelemd = ta.nelemd, np2 = ta.np2, nlev = ta.nlev; + const DepPointsH dep_points_h(dep_points, nelemd, nlev, np2, ndim); + ko::deep_copy(dep_points_h, ta.dep_points); + if (vnode) { + const DepPointsH h(vnode, nelemd, nlev, np2, ndim); + ko::deep_copy(h, ta.vnode); + } + if (vdep) { + const DepPointsH h(vdep, nelemd, nlev, np2, ndim); + ko::deep_copy(h, ta.vdep); + } +# ifdef COMPOSE_HORIZ_OPENMP + } +# pragma omp barrier +# endif +#endif +} + +template +void sl_h2d (TracerArrays& ta, bool transfer, Real* dep_points, Int ndim) { +#if defined COMPOSE_PORT +# if defined COMPOSE_HORIZ_OPENMP +# pragma omp master + { +# endif + ko::fence(); + ta.alloc_if_not(); + const Int nelemd = ta.nelemd, qsize = ta.qsize, np2 = ta.np2, nlev = ta.nlev; + const DepPointsH cart_h(dep_points, nelemd, nlev, np2, ndim); + ko::deep_copy(ta.dep_points, cart_h); + if (transfer) { + const auto qdp_m = ko::create_mirror_view(ta.qdp); + const auto dp_m = ko::create_mirror_view(ta.dp); + const auto q_m = ko::create_mirror_view(ta.q); + for (Int ie = 0; ie < nelemd; ++ie) + for (Int iq = 0; iq < qsize; ++iq) + for (Int k = 0; k < np2; ++k) + for (Int lev = 0; lev < nlev; ++lev) { + for (Int qtl = 0; qtl < 2; ++qtl) + qdp_m(ie,qtl,iq,k,lev) = ta.pqdp(ie,qtl,iq,k,lev); + q_m(ie,iq,k,lev) = ta.pq(ie,iq,k,lev); + } + for (Int ie = 0; ie < nelemd; ++ie) for (Int k = 0; k < np2; ++k) - for (Int lev = 0; lev < nlev; ++lev) { - for (Int qtl = 0; qtl < 2; ++qtl) - qdp_m(ie,qtl,iq,k,lev) = ta.pqdp(ie,qtl,iq,k,lev); - q_m(ie,iq,k,lev) = ta.pq(ie,iq,k,lev); - } - for (Int ie = 0; ie < nelemd; ++ie) - for (Int k = 0; k < np2; ++k) - for (Int lev = 0; lev < nlev; ++lev) - dp_m(ie,k,lev) = ta.pdp(ie,k,lev); - ko::deep_copy(ta.qdp, qdp_m); - ko::deep_copy(ta.dp, dp_m); - ko::deep_copy(ta.q, q_m); + for (Int lev = 0; lev < nlev; ++lev) + dp_m(ie,k,lev) = ta.pdp(ie,k,lev); + ko::deep_copy(ta.qdp, qdp_m); + ko::deep_copy(ta.dp, dp_m); + ko::deep_copy(ta.q, q_m); + } +# ifdef COMPOSE_HORIZ_OPENMP + } +# pragma omp barrier +# endif #endif } template -void sl_d2h (const TracerArrays& ta, bool transfer, Cartesian3D* dep_points, +void sl_d2h (const TracerArrays& ta, bool transfer, Real* dep_points, Int ndim, Real* minq, Real* maxq) { #if defined COMPOSE_PORT if ( ! transfer) return; +# if defined COMPOSE_HORIZ_OPENMP +# pragma omp master + { +# endif ko::fence(); const auto q_m = ko::create_mirror_view(ta.q); const Int nelemd = ta.nelemd, qsize = ta.qsize, np2 = ta.np2, nlev = ta.nlev; @@ -80,13 +142,17 @@ void sl_d2h (const TracerArrays& ta, bool transfer, Cartesian3D* dep_points, for (Int k = 0; k < np2; ++k) for (Int lev = 0; lev < nlev; ++lev) ta.pq(ie,iq,k,lev) = q_m(ie,iq,k,lev); - const DepPointsH dep_points_h(reinterpret_cast(dep_points), nelemd, nlev, np2); + const DepPointsH dep_points_h(dep_points, nelemd, nlev, np2, ndim); const QExtremaH q_min_h(minq, nelemd, qsize, np2, nlev), q_max_h(maxq, nelemd, qsize, np2, nlev); ko::deep_copy(dep_points_h, ta.dep_points); ko::deep_copy(q_min_h, ta.q_min); ko::deep_copy(q_max_h, ta.q_max); +# ifdef COMPOSE_HORIZ_OPENMP + } +# pragma omp barrier +# endif #endif } @@ -94,6 +160,10 @@ template void cedr_h2d (const TracerArrays& ta, bool transfer) { #if defined COMPOSE_PORT if ( ! transfer) return; +# if defined COMPOSE_HORIZ_OPENMP +# pragma omp master + { +# endif ko::fence(); const auto dp3d_m = ko::create_mirror_view(ta.dp3d); const auto q_m = ko::create_mirror_view(ta.q); @@ -114,6 +184,10 @@ void cedr_h2d (const TracerArrays& ta, bool transfer) { ko::deep_copy(ta.dp3d, dp3d_m); ko::deep_copy(ta.q, q_m); ko::deep_copy(ta.spheremp, spheremp_m); +# ifdef COMPOSE_HORIZ_OPENMP + } +# pragma omp barrier +# endif #endif } @@ -121,6 +195,10 @@ template void cedr_d2h (const TracerArrays& ta, bool transfer) { #if defined COMPOSE_PORT if ( ! transfer) return; +# if defined COMPOSE_HORIZ_OPENMP +# pragma omp master + { +# endif ko::fence(); const auto q_m = ko::create_mirror_view(ta.q); const auto qdp_m = ko::create_mirror_view(ta.qdp); @@ -135,6 +213,10 @@ void cedr_d2h (const TracerArrays& ta, bool transfer) { ta.pqdp(ie,n1_qdp,iq,k,lev) = qdp_m(ie,n1_qdp,iq,k,lev); ta.pq(ie,iq,k,lev) = q_m(ie,iq,k,lev); } +# ifdef COMPOSE_HORIZ_OPENMP + } +# pragma omp barrier +# endif #endif } @@ -161,10 +243,14 @@ void delete_tracer_arrays () { } template struct TracerArrays; +template void sl_traj_h2d(TracerArrays& ta, + Real*, Real*, Real*, Int ndim); +template void sl_traj_d2h(const TracerArrays& ta, + Real*, Real*, Real*, Int ndim); template void sl_h2d(TracerArrays& ta, bool transfer, - Cartesian3D* dep_points); + Real* dep_points, Int ndim); template void sl_d2h(const TracerArrays& ta, bool transfer, - Cartesian3D* dep_points, Real* minq, Real* maxq); + Real* dep_points, Int ndim, Real* minq, Real* maxq); template void cedr_h2d(const TracerArrays& ta, bool transfer); template void cedr_d2h(const TracerArrays& ta, bool transfer); diff --git a/components/homme/src/share/compose/compose_homme.hpp b/components/homme/src/share/compose/compose_homme.hpp index a3b40a204a9a..4f12b44fccfa 100644 --- a/components/homme/src/share/compose/compose_homme.hpp +++ b/components/homme/src/share/compose/compose_homme.hpp @@ -22,7 +22,7 @@ template using FA4 = ko::View using FA5 = ko::View; template using DepPoints = - ko::View; + ko::View; template using QExtrema = ko::View; @@ -140,26 +140,26 @@ struct HommeFormatArray { COMPOSE_FORCEINLINE_FUNCTION T& operator() (const Int& ie, const Int& i) const { static_assert(rank == 2, "rank 2 array"); - assert(i >= 0); - assert(ie_data_ptr[ie]); // These routines are not used on the GPU, but they can be called from // KOKKOS_FUNCTIONs on CPU in GPU builds. Avoid nvcc warnings as follows: #if defined __CUDA_ARCH__ || defined __HIP_DEVICE_COMPILE__ return unused(); #else + assert(i >= 0); + assert(ie_data_ptr[ie]); return *(ie_data_ptr[ie] + i); #endif } COMPOSE_FORCEINLINE_FUNCTION T& operator() (const Int& ie, const Int& k, const Int& lev) const { static_assert(rank == 3, "rank 3 array"); +#if defined __CUDA_ARCH__ || defined __HIP_DEVICE_COMPILE__ + return unused(); +#else assert(k >= 0); assert(lev >= 0); assert(ie_data_ptr[ie]); check(ie, k, lev); -#if defined __CUDA_ARCH__ || defined __HIP_DEVICE_COMPILE__ - return unused(); -#else return *(ie_data_ptr[ie] + lev*np2 + k); #endif } @@ -167,14 +167,14 @@ struct HommeFormatArray { T& operator() (const Int& ie, const Int& q_or_timelev, const Int& k, const Int& lev) const { static_assert(rank == 4, "rank 4 array"); +#if defined __CUDA_ARCH__ || defined __HIP_DEVICE_COMPILE__ + return unused(); +#else assert(q_or_timelev >= 0); assert(k >= 0); assert(lev >= 0); assert(ie_data_ptr[ie]); check(ie, k, lev, q_or_timelev); -#if defined __CUDA_ARCH__ || defined __HIP_DEVICE_COMPILE__ - return unused(); -#else return *(ie_data_ptr[ie] + (q_or_timelev*nlev + lev)*np2 + k); #endif } @@ -182,15 +182,15 @@ struct HommeFormatArray { T& operator() (const Int& ie, const Int& timelev, const Int& q, const Int& k, const Int& lev) const { static_assert(rank == 5, "rank 4 array"); +#if defined __CUDA_ARCH__ || defined __HIP_DEVICE_COMPILE__ + return unused(); +#else assert(timelev >= 0); assert(q >= 0); assert(k >= 0); assert(lev >= 0); assert(ie_data_ptr[ie]); check(ie, k, lev, q, timelev); -#if defined __CUDA_ARCH__ || defined __HIP_DEVICE_COMPILE__ - return unused(); -#else return *(ie_data_ptr[ie] + ((timelev*qsized + q)*nlev + lev)*np2 + k); #endif } @@ -255,7 +255,7 @@ struct TracerArrays { View dp3d; // elem%state%dp3d or the sl3d equivalent View qdp; // elem%state%Qdp(:,:,:,:,:) View q; // elem%state%Q - DepPoints dep_points; + DepPoints dep_points, vnode, vdep; QExtrema q_min, q_max; void alloc_if_not(); #else @@ -287,10 +287,18 @@ subview_ie (const Int ie, const TracerView& s) { return TracerView(&s(ie,0,0,0,0), s.extent(1), s.extent(2), s.extent(3), s.extent(4)); } template -void sl_h2d(TracerArrays& ta, bool transfer, Cartesian3D* dep_points); +void sl_traj_h2d(TracerArrays& ta, Real* dep_points, Real* vnode, Real* vdep, + Int ndim); + +template +void sl_traj_d2h(const TracerArrays& ta, Real* dep_points, Real* vnode, + Real* vdep, Int ndim); + +template +void sl_h2d(TracerArrays& ta, bool transfer, Real* dep_points, Int ndim); template -void sl_d2h(const TracerArrays& ta, bool transfer, Cartesian3D* dep_points, +void sl_d2h(const TracerArrays& ta, bool transfer, Real* dep_points, Int ndim, Real* minq, Real* maxq); template diff --git a/components/homme/src/share/compose/compose_hommexx.cpp b/components/homme/src/share/compose/compose_hommexx.cpp index fdc6cdf4bcf5..0b88e6024a1d 100644 --- a/components/homme/src/share/compose/compose_hommexx.cpp +++ b/components/homme/src/share/compose/compose_hommexx.cpp @@ -21,10 +21,13 @@ template using View = typename TracerArrays::View; #endif -void set_views (const SetView& spheremp, - const SetView& dp, const SetView& dp3d, - const SetView& qdp, const SetView& q, - const SetView& dep_points) { +void set_views (const SetView& spheremp, + const SetView& dp, const SetView5& dp3d, + const SetView& qdp, const SetView5& q, + const SetView5& dep_points, const SetView5& vnode, + const SetView5& vdep, const Int ndim) { + static_assert(std::is_same::value, + "Hommexx and Compose real types must be the same."); #ifdef COMPOSE_PORT auto& ta = *get_tracer_arrays(); const auto nel = spheremp.extent_int(0); @@ -35,13 +38,29 @@ void set_views (const SetView& spheremp, ta.dp3d = View(dp3d.data(), nel, dp3d.extent_int(1), np2, nlev); ta.qdp = View(qdp.data(), nel, qdp.extent_int(1), qdp.extent_int(2), np2, nlev); ta.q = View(q.data(), nel, q.extent_int(1), np2, nlev); - ta.dep_points = View(dep_points.data(), nel, dep_points.extent_int(1), np2); + ta.dep_points = View(dep_points.data(), nel, dep_points.extent_int(1), np2, ndim); + if (vnode.data()) + ta.vnode = View(vnode.data(), nel, vnode.extent_int(1), np2, ndim); + if (vdep.data()) + ta.vdep = View(vdep.data(), nel, vdep .extent_int(1), np2, ndim); #else slmm_throw_if(true, "Running a Hommexx code path with the non-Hommexx build" " is not supported.\n"); #endif } +void set_hvcoord (const HommexxReal etai_beg, const HommexxReal etai_end, + const HommexxReal* etam) { + auto& cm = *get_isl_mpi_singleton(); + islmpi::set_hvcoord(cm, etai_beg, etai_end, etam); +} + +void calc_v_departure (const int step, const HommexxReal dtsub) { + auto& cm = *get_isl_mpi_singleton(); + islmpi::calc_v_departure<>(cm, 0, cm.nelemd - 1, step, dtsub, + nullptr, nullptr, nullptr); +} + void advect (const int np1, const int n0_qdp, const int np1_qdp) { auto& cm = *get_isl_mpi_singleton(); cm.tracer_arrays->np1 = np1; diff --git a/components/homme/src/share/compose/compose_hommexx.hpp b/components/homme/src/share/compose/compose_hommexx.hpp index 6cd8f510e8e2..b2e339055404 100644 --- a/components/homme/src/share/compose/compose_hommexx.hpp +++ b/components/homme/src/share/compose/compose_hommexx.hpp @@ -6,13 +6,22 @@ namespace homme { namespace compose { +typedef double HommexxReal; + template using SetView = Kokkos::View; -void set_views(const SetView& spheremp, - const SetView& dp, const SetView& dp3d, - const SetView& qdp, const SetView& q, - const SetView& dep_points); +typedef SetView SetView5; + +void set_views(const SetView& spheremp, + const SetView& dp, const SetView5& dp3d, + const SetView& qdp, const SetView5& q, + const SetView5& dep_points, const SetView5& vnode, + const SetView5& vdep, const int trajectory_ndim); + +void set_hvcoord(const HommexxReal etai_beg, const HommexxReal etai_end, + const HommexxReal* etam); +void calc_v_departure(const int step, const HommexxReal dtsub); void advect(const int np1, const int n0_qdp, const int np1_qdp); diff --git a/components/homme/src/share/compose/compose_slmm.cpp b/components/homme/src/share/compose/compose_slmm.cpp index 27ab1e1a7d04..ec570e21580a 100644 --- a/components/homme/src/share/compose/compose_slmm.cpp +++ b/components/homme/src/share/compose/compose_slmm.cpp @@ -98,11 +98,12 @@ init (const typename IslMpi::Advecter::ConstPtr& advecter, const mpi::Parallel::Ptr& p, Int np, Int nlev, Int qsize, Int qsized, Int nelemd, const Int* nbr_id_rank, const Int* nirptr, - Int halo) { - slmm_throw_if(halo < 1 || halo > 2, "halo must be 1 (default) or 2."); + Int halo, Int traj_3d, Int traj_nsubstep) { + slmm_throw_if(halo < 1, "halo must be 1 (default) or larger."); auto tracer_arrays = homme::init_tracer_arrays(nelemd, nlev, np, qsize, qsized); auto cm = std::make_shared >(p, advecter, tracer_arrays, np, nlev, - qsize, qsized, nelemd, halo); + qsize, qsized, nelemd, halo, traj_3d, + traj_nsubstep); setup_comm_pattern(*cm, nbr_id_rank, nirptr); return cm; } @@ -111,13 +112,43 @@ init (const typename IslMpi::Advecter::ConstPtr& advecter, // already has a ref to the const'ed one. template void finalize_init_phase (IslMpi& cm, typename IslMpi::Advecter& advecter) { - if (cm.halo == 2) + if (cm.halo > 1) extend_halo::extend_local_meshes(*cm.p, cm.ed_h, advecter); advecter.fill_nearest_points_if_needed(); advecter.sync_to_device(); sync_to_device(cm); } +template +void set_hvcoord (IslMpi& cm, const Real etai_beg, const Real etai_end, + const Real* etam) { + if (cm.etam.size() > 0) return; +#if defined COMPOSE_HORIZ_OPENMP +# pragma omp barrier +# pragma omp master +#endif + { + slmm_assert(cm.nlev > 0); + cm.etai_beg = etai_beg; + cm.etai_end = etai_end; + cm.etam = typename IslMpi::template ArrayD("etam", cm.nlev); + const auto h = ko::create_mirror_view(cm.etam); + for (int k = 0; k < cm.nlev; ++k) { + h(k) = etam[k]; + slmm_assert(k == 0 || h(k) > h(k-1)); + slmm_assert(h(k) > 0 && h(k) < 1); + } + ko::deep_copy(cm.etam, h); + } +#if defined COMPOSE_HORIZ_OPENMP +# pragma omp barrier +#endif +} + +template void set_hvcoord( + IslMpi& cm, const Real etai_beg, const Real etai_end, + const Real* etam); + // Set pointers to HOMME data arrays. template void set_elem_data (IslMpi& cm, const Int ie, Real* qdp, const Int n0_qdp, @@ -297,8 +328,9 @@ void slmm_init_impl ( homme::Int nelemd, homme::Int cubed_sphere_map, homme::Int geometry, const homme::Int* lid2gid, const homme::Int* lid2facenum, const homme::Int* nbr_id_rank, const homme::Int* nirptr, - homme::Int sl_nearest_point_lev, homme::Int, homme::Int, homme::Int, - homme::Int) + homme::Int sl_halo, homme::Int sl_traj_3d, homme::Int sl_traj_nsubstep, + homme::Int sl_nearest_point_lev, + homme::Int, homme::Int, homme::Int, homme::Int) { amb::dev_init_threads(); homme::slmm_init(np, nelem, nelemd, transport_alg, cubed_sphere_map, @@ -308,7 +340,7 @@ void slmm_init_impl ( const auto p = homme::mpi::make_parallel(MPI_Comm_f2c(fcomm)); homme::g_csl_mpi = homme::islmpi::init( homme::g_advecter, p, np, nlev, qsize, qsized, nelemd, - nbr_id_rank, nirptr, 2 /* halo */); + nbr_id_rank, nirptr, sl_halo, sl_traj_3d, sl_traj_nsubstep); amb::dev_fin_threads(); } @@ -374,6 +406,42 @@ void slmm_check_ref2sphere (homme::Int ie, homme::Cartesian3D* p) { amb::dev_fin_threads(); } +void slmm_set_hvcoord (const homme::Real etai_beg, const homme::Real etai_end, + const homme::Real* etam) { + amb::dev_init_threads(); + slmm_assert(homme::g_csl_mpi); + homme::islmpi::set_hvcoord(*homme::g_csl_mpi, etai_beg, etai_end, etam); + amb::dev_fin_threads(); +} + +void slmm_calc_v_departure ( + homme::Int nets, homme::Int nete, homme::Int step, homme::Real dtsub, + homme::Real* dep_points, homme::Int dep_points_ndim, homme::Real* vnode, + homme::Real* vdep, homme::Int* info) +{ + amb::dev_init_threads(); + check_threading(); + slmm_assert(homme::g_csl_mpi); + slmm_assert(homme::g_csl_mpi->sendsz.empty()); // alloc_mpi_buffers was called + auto& cm = *homme::g_csl_mpi; + slmm_assert(cm.dep_points_ndim == dep_points_ndim); + { + slmm::Timer timer("h2d"); + homme::sl_traj_h2d(*cm.tracer_arrays, dep_points, vnode, vdep, + cm.dep_points_ndim); + } + homme::islmpi::calc_v_departure(cm, nets - 1, nete - 1, step - 1, + dtsub, dep_points, vnode, vdep); + *info = 0; + { + slmm::Timer timer("d2h"); + homme::sl_traj_d2h(*cm.tracer_arrays, dep_points, vnode, vdep, + cm.dep_points_ndim); + } + amb::dev_fin_threads(); +} + +// Request extra data to be transferred for analysis. static bool s_h2d, s_d2h; void slmm_csl_set_elem_data ( @@ -389,34 +457,35 @@ void slmm_csl_set_elem_data ( amb::dev_fin_threads(); } -void slmm_csl ( - homme::Int nets, homme::Int nete, homme::Cartesian3D* dep_points, - homme::Real* minq, homme::Real* maxq, homme::Int* info) -{ +void slmm_csl (homme::Int nets, homme::Int nete, homme::Real* dep_points, + homme::Int dep_points_ndim, homme::Real* minq, homme::Real* maxq, + homme::Int* info) { amb::dev_init_threads(); check_threading(); slmm_assert(homme::g_csl_mpi); slmm_assert(homme::g_csl_mpi->sendsz.empty()); // alloc_mpi_buffers was called - { slmm::Timer timer("h2d"); - //if (homme::g_csl_mpi->p->amroot() && s_h2d) printf("sl_h2d\n"); - homme::sl_h2d(*homme::g_csl_mpi->tracer_arrays, s_h2d, dep_points); } + auto& cm = *homme::g_csl_mpi; + slmm_assert(cm.dep_points_ndim == dep_points_ndim); + { + slmm::Timer timer("h2d"); + homme::sl_h2d(*cm.tracer_arrays, s_h2d, dep_points, cm.dep_points_ndim); + } *info = 0; -#if 0 -#pragma message "RM TRY-CATCH WHILE DEV'ING" +#if 1 try { - homme::islmpi::step(*homme::g_csl_mpi, nets - 1, nete - 1, - reinterpret_cast(dep_points), minq, maxq); + homme::islmpi::step(cm, nets - 1, nete - 1, dep_points, minq, maxq); } catch (const std::exception& e) { std::cerr << e.what(); *info = -1; } #else - homme::islmpi::step(*homme::g_csl_mpi, nets - 1, nete - 1, - reinterpret_cast(dep_points), minq, maxq); + homme::islmpi::step(cm, nets - 1, nete - 1, dep_points, minq, maxq); #endif - { slmm::Timer timer("d2h"); - //if (homme::g_csl_mpi->p->amroot() && s_d2h) printf("sl_d2h\n"); - homme::sl_d2h(*homme::g_csl_mpi->tracer_arrays, s_d2h, dep_points, minq, maxq); } + { + slmm::Timer timer("d2h"); + homme::sl_d2h(*cm.tracer_arrays, s_d2h, dep_points, cm.dep_points_ndim, + minq, maxq); + } amb::dev_fin_threads(); } diff --git a/components/homme/src/share/compose/compose_slmm_departure_point.hpp b/components/homme/src/share/compose/compose_slmm_departure_point.hpp index de912c542160..c226787cfc81 100644 --- a/components/homme/src/share/compose/compose_slmm_departure_point.hpp +++ b/components/homme/src/share/compose/compose_slmm_departure_point.hpp @@ -1,6 +1,7 @@ #ifndef INCLUDE_COMPOSE_SLMM_DEPARTURE_POINT_HPP #define INCLUDE_COMPOSE_SLMM_DEPARTURE_POINT_HPP +#include "compose.hpp" #include "compose_slmm.hpp" namespace slmm { diff --git a/components/homme/src/share/compose/compose_slmm_islmpi.cpp b/components/homme/src/share/compose/compose_slmm_islmpi.cpp index 822780dd967d..635e020877c2 100644 --- a/components/homme/src/share/compose/compose_slmm_islmpi.cpp +++ b/components/homme/src/share/compose/compose_slmm_islmpi.cpp @@ -92,20 +92,23 @@ typedef std::vector GidRankPairs; typedef std::map Gid2Nbrs; typedef std::vector IntBuf; typedef std::vector RealBuf; +typedef std::map Gid2Count; template -GidRankPairs all_nbrs_but_me (const typename IslMpi::ElemDataH& ed) { +GidRankPairs all_1halo_nbrs_but_me (const typename IslMpi::ElemDataH& ed) { GidRankPairs gs; - gs.reserve(ed.nbrs.size() - 1); - for (const auto& n : ed.nbrs) + gs.reserve(ed.nin1halo - 1); + for (Int i = 0; i < ed.nin1halo; ++i) { + const auto& n = ed.nbrs(i); if (&n != ed.me) gs.push_back(GidRankPair(n.gid, n.rank)); + } return gs; } template void fill_gid2nbrs (const mpi::Parallel& p, const typename IslMpi::ElemDataListH& eds, - Gid2Nbrs& gid2nbrs) { + Gid2Nbrs& gid2nbrs, Gid2Count& gid2ninprevhalo) { static const Int tag = 6; const Rank my_rank = p.rank(); const Int n_owned = eds.size(); @@ -113,7 +116,7 @@ void fill_gid2nbrs (const mpi::Parallel& p, const typename IslMpi::ElemDataL // Fill in the ones we know. for (const auto& ed : eds) { slmm_assert(ed.me->rank == my_rank); - gid2nbrs[ed.me->gid] = all_nbrs_but_me(ed); + gid2nbrs[ed.me->gid] = all_1halo_nbrs_but_me(ed); } std::vector ranks; @@ -126,13 +129,18 @@ void fill_gid2nbrs (const mpi::Parallel& p, const typename IslMpi::ElemDataL std::map needgid2rank; { std::set unique_ranks; - for (const auto& item : gid2nbrs) - for (const auto& n : item.second) - if (n.rank != my_rank) { - slmm_assert(gid2nbrs.find(n.gid) == gid2nbrs.end()); - needgid2rank.insert(std::make_pair(n.gid, n.rank)); - unique_ranks.insert(n.rank); - } + for (const auto& ed : eds) { + // We only need information for GIDs in the current outermost halo. + const auto& it = gid2ninprevhalo.find(ed.me->gid); + const Int i0 = it == gid2ninprevhalo.end() ? 0 : it->second; + for (Int i = i0; i < ed.nbrs.size(); ++i) { + const auto& n = ed.nbrs(i); + if (n.rank == my_rank) continue; + slmm_assert(gid2nbrs.find(n.gid) == gid2nbrs.end()); + needgid2rank.insert(std::make_pair(n.gid, n.rank)); + unique_ranks.insert(n.rank); + } + } nrank = unique_ranks.size(); ranks.insert(ranks.begin(), unique_ranks.begin(), unique_ranks.end()); Int i = 0; @@ -171,8 +179,9 @@ void fill_gid2nbrs (const mpi::Parallel& p, const typename IslMpi::ElemDataL std::vector nbr_send_reqs(nrank), nbr_recv_reqs(nrank); for (Int i = 0; i < nrank; ++i) { auto& r = nbr_recvs[i]; - // 20 is from dimensions_mod::set_mesh_dimensions; factor of 2 is to get - // (gid,rank); 1 is for size datum. + // 20 is from dimensions_mod::set_mesh_dimensions, the maximum size of the + // 1-halo minus the 0-halo; factor of 2 is to get (gid,rank); 1 is for the + // size datum. r.resize((20*2 + 1)*(req_sends[i].size() - 1)); mpi::irecv(p, r.data(), r.size(), ranks[i], tag, &nbr_recv_reqs[i]); } @@ -215,17 +224,22 @@ void fill_gid2nbrs (const mpi::Parallel& p, const typename IslMpi::ElemDataL } template -void extend_nbrs (const Gid2Nbrs& gid2nbrs, typename IslMpi::ElemDataListH& eds) { +void extend_nbrs (const Gid2Nbrs& gid2nbrs, typename IslMpi::ElemDataListH& eds, + Gid2Count& gid2ninprevhalo) { for (auto& ed : eds) { - // Get all <=2-halo neighbors. - std::set new_nbrs; - for (const auto& n : ed.nbrs) { - if (&n == ed.me) continue; - const auto& it = gid2nbrs.find(n.gid); - slmm_assert(it != gid2nbrs.end()); - const auto& gid_nbrs = it->second; - for (const auto& gn : gid_nbrs) - new_nbrs.insert(gn); + // Get all <=(n+1)-halo neighbors, where we already have <=n-halo neighbors. + std::set new_nbrs; { + const auto& it = gid2ninprevhalo.find(ed.me->gid); + const Int i0 = it == gid2ninprevhalo.end() ? 0 : it->second; + for (Int i = i0; i < ed.nbrs.size(); ++i) { + const auto& n = ed.nbrs(i); + if (&n == ed.me) continue; + const auto& it = gid2nbrs.find(n.gid); + slmm_assert(it != gid2nbrs.end()); + const auto& gid_nbrs = it->second; + for (const auto& gn : gid_nbrs) + new_nbrs.insert(gn); + } } // Remove the already known ones. for (const auto& n : ed.nbrs) @@ -238,8 +252,9 @@ void extend_nbrs (const Gid2Nbrs& gid2nbrs, typename IslMpi::ElemDataListH& break; } slmm_assert(me >= 0); - // Append the, now only new, 2-halo ones. + // Append the, now only new, (n+1)-halo ones. Int i = ed.nbrs.size(); + gid2ninprevhalo[ed.me->gid] = i; ed.nbrs.reset_capacity(i + new_nbrs.size(), true); ed.me = &ed.nbrs(me); for (const auto& n : new_nbrs) { @@ -250,14 +265,22 @@ void extend_nbrs (const Gid2Nbrs& gid2nbrs, typename IslMpi::ElemDataListH& en.lid_on_rank = -1; en.lid_on_rank_idx = -1; } +#ifndef NDEBUG + { + std::set ugid; + for (Int i = 0; i < ed.nbrs.size(); ++i) ugid.insert(ed.nbrs(i).gid); + slmm_assert(ugid.size() == size_t(ed.nbrs.size())); + } +#endif } } template -void collect_gid_rank (const mpi::Parallel& p, typename IslMpi::ElemDataListH& eds) { +void collect_gid_rank (const mpi::Parallel& p, typename IslMpi::ElemDataListH& eds, + Gid2Count& gid2ninprevhalo) { Gid2Nbrs gid2nbrs; - fill_gid2nbrs(p, eds, gid2nbrs); - extend_nbrs(gid2nbrs, eds); + fill_gid2nbrs(p, eds, gid2nbrs, gid2ninprevhalo); + extend_nbrs(gid2nbrs, eds, gid2ninprevhalo); } template @@ -478,7 +501,11 @@ void collect_gid_rank (IslMpi& cm, const Int* nbr_id_rank, const Int* nirptr } slmm_assert(ed.me); } - if (cm.halo == 2) extend_halo::collect_gid_rank(*cm.p, cm.ed_h); + if (cm.halo > 1) { + extend_halo::Gid2Count gid2ninprevhalo; + for (int halo = 2; halo <= cm.halo; ++halo) + extend_halo::collect_gid_rank(*cm.p, cm.ed_h, gid2ninprevhalo); + } #ifdef COMPOSE_PORT cm.own_dep_mask = typename IslMpi::DepMask("own_dep_mask", cm.nelemd, cm.nlev, cm.np2); @@ -682,17 +709,20 @@ void size_mpi_buffers (IslMpi& cm, const Rank2Gids& rank2rmtgids, const Int sor = sizeof(Real), soi = sizeof(Int), sosi = sor; static_assert(sizeof(Real) >= sizeof(Int), "For buffer packing, we require sizeof(Real) >= sizeof(Int)"); + const bool calc_trajectory = cm.traj_nsubstep > 0; + const Int ndim = calc_trajectory ? cm.dep_points_ndim : 3; + const Int qsize = calc_trajectory ? std::max(cm.dep_points_ndim, cm.qsize) : cm.qsize; const auto xbufcnt = [&] (const std::set& rmtgids, const std::set& owngids, const bool include_bulk = true) -> Int { - return (sosi + (2*soi + (2*soi)*cm.nlev)*rmtgids.size() + // meta data - (include_bulk ? 1 : 0)*owngids.size()*cm.nlev*cm.np2*3*sor); // bulk data + return (sosi + (2*soi + (2*soi)*cm.nlev)*rmtgids.size() + // meta data + (include_bulk ? 1 : 0)*owngids.size()*cm.nlev*cm.np2*ndim*sor); // bulk data }; const auto qbufcnt = [&] (const std::set& rmtgids, const std::set& owngids) -> Int { return ((rmtgids.size()*2 + // min/max q owngids.size()*cm.np2)* // q - cm.qsize*cm.nlev*sor); + qsize*cm.nlev*sor); }; const auto bytes2real = [&] (const Int& bytes) { return (bytes + sor - 1)/sor; diff --git a/components/homme/src/share/compose/compose_slmm_islmpi.hpp b/components/homme/src/share/compose/compose_slmm_islmpi.hpp index ef30c826acf2..1e60d602dea1 100644 --- a/components/homme/src/share/compose/compose_slmm_islmpi.hpp +++ b/components/homme/src/share/compose/compose_slmm_islmpi.hpp @@ -11,7 +11,9 @@ #include // AMB 2017/06-2020/05 Initial for E3SMv2 -// AMB 2020/05-? Performance-portable impl +// AMB 2020/05-2021/01 Performance-portable impl +// AMB 2021/04 Support doubly-periodic planar mode +// AMB 2024/04-2025/01 Enhanced trajectory method namespace homme { namespace mpi { //todo Share with cedr. @@ -20,14 +22,14 @@ class Parallel { MPI_Comm comm_; public: typedef std::shared_ptr Ptr; - Parallel(MPI_Comm comm) : comm_(comm) {} + Parallel (MPI_Comm comm) : comm_(comm) {} MPI_Comm comm () const { return comm_; } - Int size() const { + Int size () const { int sz = 0; MPI_Comm_size(comm_, &sz); return sz; } - Int rank() const { + Int rank () const { int pid = 0; MPI_Comm_rank(comm_, &pid); return pid; @@ -85,6 +87,12 @@ int irecv (const Parallel& p, T* buf, int count, int src, int tag, Request* ireq int waitany(int count, Request* reqs, int* index, MPI_Status* stats = nullptr); int waitall(int count, Request* reqs, MPI_Status* stats = nullptr); int wait(Request* req, MPI_Status* stat = nullptr); + +template +int all_reduce (const Parallel& p, const T* sendbuf, T* rcvbuf, int count, MPI_Op op) { + MPI_Datatype dt = get_type(); + return MPI_Allreduce(const_cast(sendbuf), rcvbuf, count, dt, op, p.comm()); +} } // namespace mpi namespace islmpi { @@ -251,6 +259,40 @@ void deep_copy (FixedCapList& d, const FixedCapList& s) { #endif } +template +struct FixedCapListHostOnly { + FixedCapListHostOnly (const Int cap = 0) { + slmm_assert_high(cap >= 0); + reset_capacity(cap); + } + + void reset_capacity (const Int cap, const bool also_size = false) { + slmm_assert(cap >= 0); + d_.resize(cap); + n_ = also_size ? cap : 0; + } + + Int capacity () const { return d_.size(); } + Int size () const { return n_; } + Int n () const { return n_; } + + void clear () { n_ = 0; } + + void inc () { ++n_; slmm_kernel_assert_high(n_ <= static_cast(d_.size())); } + void inc (const Int& dn) { n_ += dn; slmm_kernel_assert_high(n_ <= static_cast(d_.size())); } + + T& operator() (const Int& i) { slmm_kernel_assert_high(i >= 0 && i < n_); return d_[i]; } + + T* data () { return d_.data(); } + T& back () { slmm_kernel_assert_high(n_ > 0); return d_[n_-1]; } + T* begin () { return d_.data(); } + T* end () { return d_.data() + n_; } + +private: + std::vector d_; + Int n_; +}; + template struct BufferLayoutArray; template @@ -512,6 +554,11 @@ struct IslMpi { const mpi::Parallel::Ptr p; const typename Advecter::ConstPtr advecter; const Int np, np2, nlev, qsize, qsized, nelemd, halo; + const bool traj_3d; + const Int traj_nsubstep, dep_points_ndim; + + Real etai_beg, etai_end; + ArrayD etam; ElemDataListH ed_h; // this rank's owned cells, indexed by LID ElemDataListD ed_d; @@ -526,7 +573,7 @@ struct IslMpi { BufferLayoutArray bla; // MPI comm data. - FixedCapList sendreq, recvreq; + FixedCapListHostOnly sendreq, recvreq; FixedCapList recvreq_ri; ListOfLists sendbuf, recvbuf; #ifdef COMPOSE_MPI_ON_HOST @@ -559,11 +606,14 @@ struct IslMpi { Int own_dep_list_len; IslMpi (const mpi::Parallel::Ptr& ip, const typename Advecter::ConstPtr& advecter, - const typename TracerArrays::Ptr& tracer_arrays_, - Int inp, Int inlev, Int iqsize, Int iqsized, Int inelemd, Int ihalo) + const typename TracerArrays::Ptr& itracer_arrays, + Int inp, Int inlev, Int iqsize, Int iqsized, Int inelemd, Int ihalo, + Int itraj_3d, Int itraj_nsubstep) : p(ip), advecter(advecter), np(inp), np2(np*np), nlev(inlev), qsize(iqsize), qsized(iqsized), nelemd(inelemd), - halo(ihalo), tracer_arrays(tracer_arrays_) + halo(ihalo), traj_3d(itraj_3d), traj_nsubstep(itraj_nsubstep), + dep_points_ndim(traj_3d && traj_nsubstep > 0 ? 4 : 3), + tracer_arrays(itracer_arrays) {} IslMpi(const IslMpi&) = delete; @@ -635,16 +685,17 @@ void wait_on_send (IslMpi& cm, const bool skip_if_empty = false); template void recv(IslMpi& cm, const bool skip_if_empty = false); -const int nreal_per_2int = (2*sizeof(Int) + sizeof(Real) - 1) / sizeof(Real); - template -void pack_dep_points_sendbuf_pass1(IslMpi& cm); +void pack_dep_points_sendbuf_pass1(IslMpi& cm, const bool trajectory = false); template -void pack_dep_points_sendbuf_pass2(IslMpi& cm, const DepPoints& dep_points); +void pack_dep_points_sendbuf_pass2(IslMpi& cm, const DepPoints& dep_points, + const bool trajectory = false); template void calc_q_extrema(IslMpi& cm, const Int& nets, const Int& nete); +template +void calc_rmt_q_pass1(IslMpi& cm, const bool trajectory = false); template void calc_rmt_q(IslMpi& cm); template @@ -681,8 +732,17 @@ void copy_q(IslMpi& cm, const Int& nets, template void step( IslMpi& cm, const Int nets, const Int nete, - Real* dep_points_r, // dep_points(1:3, 1:np, 1:np) - Real* q_min_r, Real* q_max_r); // q_{min,max}(1:np, 1:np, lev, 1:qsize, ie-nets+1) + Real* dep_points_r, + Real* q_min_r, Real* q_max_r); + +template +void set_hvcoord(IslMpi& cm, const Real etai_beg, const Real etai_end, + const Real* etam); + +template +void calc_v_departure( + IslMpi& cm, const Int nets, const Int nete, const Int step, const Real dtsub, + Real* dep_points_r, const Real* vnode, Real* vdep); } // namespace islmpi } // namespace homme diff --git a/components/homme/src/share/compose/compose_slmm_islmpi_buf.hpp b/components/homme/src/share/compose/compose_slmm_islmpi_buf.hpp new file mode 100644 index 000000000000..9cf8b0fe61f2 --- /dev/null +++ b/components/homme/src/share/compose/compose_slmm_islmpi_buf.hpp @@ -0,0 +1,53 @@ +#ifndef INCLUDE_COMPOSE_SLMM_ISLMPI_BUF_HPP +#define INCLUDE_COMPOSE_SLMM_ISLMPI_BUF_HPP + +namespace homme { +namespace islmpi { + +const int nreal_per_2int = (2*sizeof(Int) + sizeof(Real) - 1) / sizeof(Real); + +template SLMM_KIF +Int setbuf (Buffer& buf, const Int& os, const Int& i1, const Int& i2) { + Int* const b = reinterpret_cast(&buf(os)); + b[0] = i1; + b[1] = i2; + return nreal_per_2int; +} + +template SLMM_KIF +Int setbuf (Buffer& buf, const Int& os, const Int& i1, const short& i2, const short& i3) { + static_assert(sizeof(Int) >= 2*sizeof(short), "Need >= 2 shorts per Int"); + Int* const b = reinterpret_cast(&buf(os)); + b[0] = i1; + short* const b2 = reinterpret_cast(b+1); + b2[0] = i2; + b2[1] = i3; + return nreal_per_2int; +} + +template SLMM_KIF +Int setbuf (Buffer& buf, const Int& os, const Int& i1, const Int& i2, + const bool final) { + if (final) setbuf(buf, os, i1, i2); + return nreal_per_2int; +} + +template SLMM_KIF +Int setbuf (Buffer& buf, const Int& os, const Int& i1, const short& i2, const short& i3, + const bool final) { + if (final) setbuf(buf, os, i1, i2, i3); + return nreal_per_2int; +} + +template SLMM_KIF +Int getbuf (Buffer& buf, const Int& os, Int& i1, Int& i2) { + const Int* const b = reinterpret_cast(&buf(os)); + i1 = b[0]; + i2 = b[1]; + return nreal_per_2int; +} + +} // namespace islmpi +} // namespace homme + +#endif diff --git a/components/homme/src/share/compose/compose_slmm_islmpi_calc_trajectory.cpp b/components/homme/src/share/compose/compose_slmm_islmpi_calc_trajectory.cpp new file mode 100644 index 000000000000..273a7e80e9cb --- /dev/null +++ b/components/homme/src/share/compose/compose_slmm_islmpi_calc_trajectory.cpp @@ -0,0 +1,333 @@ +#include "compose_slmm_islmpi.hpp" +#include "compose_slmm_islmpi_interpolate.hpp" +#include "compose_slmm_islmpi_buf.hpp" + +namespace homme { +namespace islmpi { + +template using CA4 = ko::View; + +template SLMM_KF void +interpolate_vertical (const Int nlev, const Real etai_beg, const Real etai_end, + const EtamT& etam, const VnodeT& vnode, + const Int src_lid, const Int lev, const Real eta_dep, + const Real rx[np], const Real ry[np], Real* const v_tgt) { + slmm_kernel_assert(eta_dep > etai_beg && eta_dep < etai_end); + + // Search for the eta midpoint values that support the departure point's eta + // value. + Int lev_dep = lev; + if (eta_dep != etam(lev)) { + if (eta_dep < etam(lev)) { + for (lev_dep = lev-1; lev_dep >= 0; --lev_dep) + if (eta_dep >= etam(lev_dep)) + break; + } else { + for (lev_dep = lev; lev_dep < nlev-1; ++lev_dep) + if (eta_dep < etam(lev_dep+1)) + break; + } + } + slmm_kernel_assert(lev_dep >= -1 && lev_dep < nlev); + slmm_kernel_assert(lev_dep == -1 || eta_dep >= etam(lev_dep)); + Real a; + bool bdy = false; + if (lev_dep == -1) { + lev_dep = 0; + a = 0; + bdy = true; + } else if (lev_dep == nlev-1) { + a = 0; + bdy = true; + } else { + a = ((eta_dep - etam(lev_dep)) / + (etam(lev_dep+1) - etam(lev_dep))); + } + // Linear interp coefficients. + const Real alpha[] = {1-a, a}; + + for (int d = 0; d < 4; ++d) + v_tgt[d] = 0; + for (int i = 0; i < 2; ++i) { + if (alpha[i] == 0) continue; + for (int d = 0; d < 4; ++d) { + Real vel_nodes[np*np]; + for (int k = 0; k < np*np; ++k) + vel_nodes[k] = vnode(src_lid,lev_dep+i,k,d); + v_tgt[d] += alpha[i]*calc_q_tgt(rx, ry, vel_nodes); + } + } + // Treat eta_dot specially since eta_dot goes to 0 at the boundaries. + if (bdy) { + slmm_kernel_assert(etam(0) > etai_beg); + slmm_kernel_assert(etam(nlev-1) < etai_end); + if (lev_dep == 0) + v_tgt[3] *= (eta_dep - etai_beg)/(etam(0) - etai_beg); + else + v_tgt[3] *= (etai_end - eta_dep)/(etai_end - etam(nlev-1)); + } +} + +template +void calc_v (const IslMpi& cm, const VnodeT& vnode, + const Int src_lid, const Int lev, + const Real* const dep_point, Real* const v_tgt) { + // Horizontal interpolation. + Real rx[np], ry[np]; { + Real ref_coord[2]; + const auto& m = cm.advecter->local_mesh(src_lid); + cm.advecter->s2r().calc_sphere_to_ref(src_lid, m, dep_point, + ref_coord[0], ref_coord[1]); + interpolate(cm.advecter->alg(), ref_coord, rx, ry); + } + + if (not cm.traj_3d) { + for (int d = 0; d < cm.dep_points_ndim; ++d) { + Real vel_nodes[np*np]; + for (int k = 0; k < np*np; ++k) + vel_nodes[k] = vnode(src_lid,lev,k,d); + v_tgt[d] = calc_q_tgt(rx, ry, vel_nodes); + } + return; + } + + // Vertical Interpolation. + slmm_kernel_assert(cm.dep_points_ndim == 4); + interpolate_vertical(cm.nlev, cm.etai_beg, cm.etai_end, cm.etam, vnode, + src_lid, lev, dep_point[3], rx, ry, v_tgt); +} + +template +struct CalcVData { + typedef slmm::Advecter Adv; + const typename Adv::LocalMeshesD local_meshes; + const typename Adv::Alg::Enum interp_alg; + const slmm::SphereToRef s2r; + const bool traj_3d; + const int dep_points_ndim; + const int nlev; + const Real etai_beg, etai_end; + const typename IslMpi::template ArrayD etam; + + CalcVData (const IslMpi& cm) + : local_meshes(cm.advecter->local_meshes()), + interp_alg(cm.advecter->alg()), + s2r(cm.advecter->s2r()), + traj_3d(cm.traj_3d), + dep_points_ndim(cm.dep_points_ndim), + nlev(cm.nlev), + etai_beg(cm.etai_beg), etai_end(cm.etai_end), + etam(cm.etam) + {} +}; + +template SLMM_KF +void calc_v (const CalcVData& cvd, const VnodeT& vnode, + const Int src_lid, const Int lev, + const Real* const dep_point, Real* const v_tgt) { + // Horizontal interpolation. + Real rx[np], ry[np]; { + Real ref_coord[2]; + const auto& m = cvd.local_meshes(src_lid); + cvd.s2r.calc_sphere_to_ref(src_lid, m, dep_point, + ref_coord[0], ref_coord[1]); + interpolate(cvd.interp_alg, ref_coord, rx, ry); + } + + if (not cvd.traj_3d) { + for (int d = 0; d < cvd.dep_points_ndim; ++d) { + Real vel_nodes[np*np]; + for (int k = 0; k < np*np; ++k) + vel_nodes[k] = vnode(src_lid,lev,k,d); + v_tgt[d] = calc_q_tgt(rx, ry, vel_nodes); + } + return; + } + + // Vertical Interpolation. + slmm_kernel_assert(cvd.dep_points_ndim == 4); + interpolate_vertical(cvd.nlev, cvd.etai_beg, cvd.etai_end, cvd.etam, vnode, + src_lid, lev, dep_point[3], rx, ry, v_tgt); +} + +template +void traj_calc_rmt_next_step (IslMpi& cm, const VnodeT& vnode) { + calc_rmt_q_pass1(cm, true); + const auto ndim = cm.dep_points_ndim; + const auto& rmt_xs = cm.rmt_xs; + const auto& sendbuf = cm.sendbuf; + const auto& recvbuf = cm.recvbuf; + CalcVData cvd(cm); +#ifdef COMPOSE_PORT + ko::parallel_for(ko::RangePolicy(0, cm.nrmt_xs), + COMPOSE_LAMBDA (const Int it) +#else +# ifdef COMPOSE_HORIZ_OPENMP +# pragma omp for +# endif + for (Int it = 0; it < cm.nrmt_xs; ++it) +#endif + { + const Int + ri = rmt_xs(5*it), lid = rmt_xs(5*it + 1), lev = rmt_xs(5*it + 2), + xos = rmt_xs(5*it + 3), vos = ndim*rmt_xs(5*it + 4); + const auto&& xs = recvbuf(ri); + auto&& v = sendbuf(ri); + calc_v(cvd, vnode, lid, lev, &xs(xos), &v(vos)); + } +#ifdef COMPOSE_PORT + ); +#endif +} + +template +void traj_calc_own_next_step (IslMpi& cm, const DepPoints& dep_points, + const VnodeT& vnode, const VdepT& vdep) { + const auto ndim = cm.dep_points_ndim; +#ifdef COMPOSE_PORT + const auto& ed_d = cm.ed_d; + const auto& own_dep_list = cm.own_dep_list; + CalcVData cvd(cm); + const auto f = COMPOSE_LAMBDA (const Int& it) { + const Int tci = own_dep_list(it,0); + const Int tgt_lev = own_dep_list(it,1); + const Int tgt_k = own_dep_list(it,2); + const auto& ed = ed_d(tci); + const Int slid = ed.nbrs(ed.src(tgt_lev, tgt_k)).lid_on_rank; + Real v_tgt[4]; + calc_v(cvd, vnode, slid, tgt_lev, &dep_points(tci,tgt_lev,tgt_k,0), v_tgt); + for (int d = 0; d < ndim; ++d) + vdep(tci,tgt_lev,tgt_k,d) = v_tgt[d]; + }; + ko::parallel_for( + ko::RangePolicy(0, cm.own_dep_list_len), f); +#else + const int tid = get_tid(); + for (Int tci = 0; tci < cm.nelemd; ++tci) { + auto& ed = cm.ed_d(tci); + const Int ned = ed.own.n(); +#ifdef COMPOSE_HORIZ_OPENMP +# pragma omp for +#endif + for (Int idx = 0; idx < ned; ++idx) { + const auto& e = ed.own(idx); + const Int slid = ed.nbrs(ed.src(e.lev, e.k)).lid_on_rank; + Real v_tgt[4]; + calc_v(cm, vnode, slid, e.lev, &dep_points(tci,e.lev,e.k,0), v_tgt); + for (int d = 0; d < ndim; ++d) + vdep(tci,e.lev,e.k,d) = v_tgt[d]; + } + } +#endif +} + +template +void traj_copy_next_step (IslMpi& cm, const VdepT& vdep) { + const auto myrank = cm.p->rank(); + const auto ndim = cm.dep_points_ndim; +#ifdef COMPOSE_PORT + const auto& mylid_with_comm = cm.mylid_with_comm_d; + const auto& ed_d = cm.ed_d; + const auto& recvbufs = cm.recvbuf; + const Int nlid = cm.mylid_with_comm_h.size(); + const Int nlev = cm.nlev, np2 = cm.np2; + const auto f = COMPOSE_LAMBDA (const Int& it) { + const Int tci = mylid_with_comm(it/(np2*nlev)); + const Int rmt_id = it % (np2*nlev); + auto& ed = ed_d(tci); + if (rmt_id >= ed.rmt.size()) return; + const auto& e = ed.rmt(rmt_id); + slmm_kernel_assert(ed.nbrs(ed.src(e.lev, e.k)).rank != myrank); + const Int ri = ed.nbrs(ed.src(e.lev, e.k)).rank_idx; + const auto&& recvbuf = recvbufs(ri); + for (int d = 0; d < ndim; ++d) + vdep(tci,e.lev,e.k,d) = recvbuf(e.q_ptr + d); + }; + ko::parallel_for(ko::RangePolicy(0, nlid*np2*nlev), f); +#else + const int tid = get_tid(); + for (Int ptr = cm.mylid_with_comm_tid_ptr_h(tid), + end = cm.mylid_with_comm_tid_ptr_h(tid+1); + ptr < end; ++ptr) { + const Int tci = cm.mylid_with_comm_d(ptr); + auto& ed = cm.ed_d(tci); + for (const auto& e: ed.rmt) { + slmm_assert(ed.nbrs(ed.src(e.lev, e.k)).rank != myrank); + const Int ri = ed.nbrs(ed.src(e.lev, e.k)).rank_idx; + const auto&& recvbuf = cm.recvbuf(ri); + for (int d = 0; d < ndim; ++d) + vdep(tci,e.lev,e.k,d) = recvbuf(e.q_ptr + d); + } + } +#endif +} + +// vnode and vdep are indexed as (ie,lev,k,dim), On entry, vnode contains nodal +// velocity data. These data are used to provide updates at departure points for +// both own and remote departure points, writing to vdep. dim = 0:2 is for the +// 3D Cartesian representation of the horizontal velocity; dim = 3 is for +// eta_dot. +template void +calc_v_departure (IslMpi& cm, const Int nets, const Int nete, + const Int step, const Real dtsub, + Real* dep_points_r, const Real* vnode_r, Real* vdep_r) +{ + const int np = 4; + + slmm_assert(cm.np == np); + slmm_assert((cm.traj_3d and cm.dep_points_ndim == 4) or + (not cm.traj_3d and cm.dep_points_ndim == 3)); +#ifdef COMPOSE_PORT + slmm_assert(nets == 0 && nete+1 == cm.nelemd); +#endif + + // If step = 0, the departure points are at the nodes and no interpolation is + // needed. calc_v_departure should not have been called; rather, the calling + // routine should use vnode instead of vdep in subsequent calculations. + slmm_assert(step > 0); + + const auto ndim = cm.dep_points_ndim; + +#ifdef COMPOSE_PORT + const auto& vnode = cm.tracer_arrays->vnode; + const auto& vdep = cm.tracer_arrays->vdep; +#else + CA4 vnode(vnode_r, cm.nelemd, cm.nlev, cm.np2, ndim); + CA4< Real> vdep (vdep_r , cm.nelemd, cm.nlev, cm.np2, ndim); +#endif + slmm_assert(vnode.extent_int(3) == ndim); + slmm_assert(vdep .extent_int(3) == ndim); + +#ifdef COMPOSE_PORT + const auto& dep_points = cm.tracer_arrays->dep_points; +#else + DepPointsH dep_points(dep_points_r, cm.nelemd, cm.nlev, cm.np2, ndim); +#endif + slmm_assert(dep_points.extent_int(3) == ndim); + + // See comments in homme::islmpi::step for details. Each substep follows + // essentially the same pattern. + if (cm.mylid_with_comm_tid_ptr_h.capacity() == 0) + init_mylid_with_comm_threaded(cm, nets, nete); + setup_irecv(cm); + analyze_dep_points(cm, nets, nete, dep_points); + pack_dep_points_sendbuf_pass1(cm, true /* trajectory */); + pack_dep_points_sendbuf_pass2(cm, dep_points, true /* trajectory */); + isend(cm); + recv_and_wait_on_send(cm); + traj_calc_rmt_next_step(cm, vnode); + Kokkos::fence(); + isend(cm, true /* want_req */, true /* skip_if_empty */); + setup_irecv(cm, true /* skip_if_empty */); + traj_calc_own_next_step(cm, dep_points, vnode, vdep); + recv(cm, true /* skip_if_empty */); + traj_copy_next_step(cm, vdep); + wait_on_send(cm, true /* skip_if_empty */); +} + +template void calc_v_departure( + IslMpi&, const Int, const Int, const Int, const Real, + Real*, const Real*, Real*); + +} // namespace islmpi +} // namespace homme diff --git a/components/homme/src/share/compose/compose_slmm_islmpi_interpolate.cpp b/components/homme/src/share/compose/compose_slmm_islmpi_interpolate.cpp new file mode 100644 index 000000000000..3076fb89e78c --- /dev/null +++ b/components/homme/src/share/compose/compose_slmm_islmpi_interpolate.cpp @@ -0,0 +1,67 @@ +#include "compose_slmm_islmpi_interpolate.hpp" + +namespace slmm { + +static Int test_gll () { + Int nerr = 0; + const Real tol = 1e2*std::numeric_limits::epsilon(); + GLL gll; + const Real* x, * wt; + for (Int np = 2; np <= 4; ++np) { + for (Int monotone_type = 0; monotone_type <= 1; ++monotone_type) { + const Basis b(np, monotone_type); + gll.get_coef(b, x, wt); + Real sum = 0; + for (Int i = 0; i < b.np; ++i) + sum += wt[i]; + if (std::abs(2 - sum) > tol) { + std::cerr << "test_gll " << np << ", " << monotone_type + << ": 2 - sum = " << 2 - sum << "\n"; + ++nerr; + } + for (Int j = 0; j < b.np; ++j) { + Real gj[GLL::np_max]; + gll.eval(b, x[j], gj); + for (Int i = 0; i < b.np; ++i) { + if (j == i) continue; + if (std::abs(gj[i]) > tol) { + std::cerr << "test_gll " << np << ", " << monotone_type << ": gj[" + << i << "] = " << gj[i] << "\n"; + ++nerr; + } + } + } + } + } + for (Int np = 2; np <= 4; ++np) { + const Basis b(np, 0); + Real a[] = {-0.9, -0.7, -0.3, 0.1, 0.2, 0.4, 0.6, 0.8}; + const Real delta = std::sqrt(std::numeric_limits::epsilon()); + for (size_t ia = 0; ia < sizeof(a)/sizeof(Real); ++ia) { + Real gj[GLL::np_max], gjp[GLL::np_max], gjm[GLL::np_max]; + gll.eval_derivative(b, a[ia], gj); + gll.eval(b, a[ia] + delta, gjp); + gll.eval(b, a[ia] - delta, gjm); + for (Int i = 0; i < b.np; ++i) { + const Real fd = (gjp[i] - gjm[i])/(2*delta); + if (std::abs(fd - gj[i]) >= delta*std::abs(gjp[i])) + ++nerr; + } + } + } + return nerr; +} + +} // namespace slmm + +namespace compose { +namespace test { + +int interpolate_unittest () { + int nerr = 0; + nerr += slmm::test_gll(); + return nerr; +} + +} // namespace test +} // namespace compose diff --git a/components/homme/src/share/compose/compose_slmm_islmpi_interpolate.hpp b/components/homme/src/share/compose/compose_slmm_islmpi_interpolate.hpp new file mode 100644 index 000000000000..29536e9ab9a8 --- /dev/null +++ b/components/homme/src/share/compose/compose_slmm_islmpi_interpolate.hpp @@ -0,0 +1,142 @@ +#ifndef INCLUDE_COMPOSE_SLMM_ISLMPI_INTERPOLATE_HPP +#define INCLUDE_COMPOSE_SLMM_ISLMPI_INTERPOLATE_HPP + +#include "compose.hpp" +#include "compose_slmm.hpp" +#include "compose_slmm_islmpi.hpp" + +namespace slmm { + +static constexpr Real sqrt5 = 2.23606797749978969641; // std::sqrt(5.0); +static constexpr Real oosqrt5 = 1.0 / sqrt5; + +SLMM_KIF void gll_np4_eval (const Real x, Real y[4]) { + static constexpr Real oo8 = 1.0/8.0; + const Real x2 = x*x; + y[0] = (1.0 - x)*(5.0*x2 - 1.0)*oo8; + y[1] = -sqrt5*oo8*(sqrt5 - 5.0*x)*(x2 - 1.0); + y[2] = -sqrt5*oo8*(sqrt5 + 5.0*x)*(x2 - 1.0); + y[3] = (1.0 + x)*(5.0*x2 - 1.0)*oo8; +} + +// Linear interp in each region. +SLMM_KIF void gll_np4_subgrid_eval_impl (const Real& x, Real y[4]) { + if (x < -oosqrt5) { + const Real alpha = (x + 1)/(1 - oosqrt5); + y[0] = 1 - alpha; + y[1] = alpha; + y[2] = 0; + y[3] = 0; + } else { + const Real alpha = (x + oosqrt5)/(2*oosqrt5); + y[0] = 0; + y[1] = 1 - alpha; + y[2] = alpha; + y[3] = 0; + } +} + +SLMM_KIF void gll_np4_subgrid_eval (const Real& x, Real y[4]) { + if (x > 0) { + gll_np4_subgrid_eval_impl(-x, y); + ko::swap(y[0], y[3]); + ko::swap(y[1], y[2]); + return; + } + gll_np4_subgrid_eval_impl(x, y); +} + +// Quadratic interpolant across nodes 1,2,3 -- i.e., excluding node 0 -- of the +// np=4 reference element. +SLMM_KIF void outer_eval (const Real& x, Real v[4]) { + static constexpr Real + xbar = (2*oosqrt5) / (1 + oosqrt5), + ooxbar = 1 / xbar, + ybar = 1 / (xbar - 1); + const Real xn = (x + oosqrt5) / (1 + oosqrt5); + v[0] = 0; + v[1] = 1 + ybar*xn*((1 - ooxbar)*xn + ooxbar - xbar); + v[2] = ybar*ooxbar*xn*(xn - 1); + v[3] = ybar*xn*(xbar - xn); +} + +// In the middle region, use the standard GLL np=4 interpolant; in the two outer +// regions, use an order-reduced interpolant that stabilizes the method. +SLMM_KIF void gll_np4_subgrid_exp_eval (const Real& x, Real y[4]) { + static constexpr Real + alpha = 0.5527864045000416708, + v = 0.427*(1 + alpha), + x2 = 0.4472135954999579277, + x3 = 1 - x2, + det = x2*x3*(x2 - x3), + y2 = alpha, + y3 = v, + c1 = (x3*y2 - x2*y3)/det, + c2 = (-x3*x3*y2 + x2*x2*y3)/det; + if (x < -oosqrt5 || x > oosqrt5) { + if (x < -oosqrt5) { + outer_eval(-x, y); + ko::swap(y[0], y[3]); + ko::swap(y[1], y[2]); + } else + outer_eval(x, y); + Real y4[4]; + gll_np4_eval(x, y4); + const Real x0 = 1 - std::abs(x); + const Real a = (c1*x0 + c2)*x0; + for (int i = 0; i < 4; ++i) + y[i] = a*y[i] + (1 - a)*y4[i]; + } else + gll_np4_eval(x, y); +} + +} // namespace slmm + +namespace homme { +namespace islmpi { + +template +SLMM_KIF void interpolate (const typename IslMpi::Advecter::Alg::Enum& alg, + const Real ref_coord[2], Real rx[4], Real ry[4]) { + typedef typename IslMpi::Advecter::Alg Alg; + switch (alg) { + case Alg::csl_gll: + slmm::gll_np4_eval(ref_coord[0], rx); + slmm::gll_np4_eval(ref_coord[1], ry); + break; + case Alg::csl_gll_subgrid: + slmm::gll_np4_subgrid_eval(ref_coord[0], rx); + slmm::gll_np4_subgrid_eval(ref_coord[1], ry); + break; + case Alg::csl_gll_exp: + slmm::gll_np4_subgrid_exp_eval(ref_coord[0], rx); + slmm::gll_np4_subgrid_exp_eval(ref_coord[1], ry); + break; + default: + slmm_kernel_assert(0); + } +} + +SLMM_KIF Real calc_q_tgt (const Real rx[4], const Real ry[4], const Real qs[16]) { + return (ry[0]*(rx[0]*qs[ 0] + rx[1]*qs[ 1] + rx[2]*qs[ 2] + rx[3]*qs[ 3]) + + ry[1]*(rx[0]*qs[ 4] + rx[1]*qs[ 5] + rx[2]*qs[ 6] + rx[3]*qs[ 7]) + + ry[2]*(rx[0]*qs[ 8] + rx[1]*qs[ 9] + rx[2]*qs[10] + rx[3]*qs[11]) + + ry[3]*(rx[0]*qs[12] + rx[1]*qs[13] + rx[2]*qs[14] + rx[3]*qs[15])); +} + +SLMM_KIF Real calc_q_tgt (const Real rx[4], const Real ry[4], const Real qdp[16], + const Real dp[16]) { + return (ry[0]*(rx[0]*(qdp[ 0]/dp[ 0]) + rx[1]*(qdp[ 1]/dp[ 1]) + + rx[2]*(qdp[ 2]/dp[ 2]) + rx[3]*(qdp[ 3]/dp[ 3])) + + ry[1]*(rx[0]*(qdp[ 4]/dp[ 4]) + rx[1]*(qdp[ 5]/dp[ 5]) + + rx[2]*(qdp[ 6]/dp[ 6]) + rx[3]*(qdp[ 7]/dp[ 7])) + + ry[2]*(rx[0]*(qdp[ 8]/dp[ 8]) + rx[1]*(qdp[ 9]/dp[ 9]) + + rx[2]*(qdp[10]/dp[10]) + rx[3]*(qdp[11]/dp[11])) + + ry[3]*(rx[0]*(qdp[12]/dp[12]) + rx[1]*(qdp[13]/dp[13]) + + rx[2]*(qdp[14]/dp[14]) + rx[3]*(qdp[15]/dp[15]))); +} + +} // namespace islmpi +} // namespace homme + +#endif diff --git a/components/homme/src/share/compose/compose_slmm_islmpi_pack.cpp b/components/homme/src/share/compose/compose_slmm_islmpi_pack.cpp index 4fe325728dfa..213226b6c3b8 100644 --- a/components/homme/src/share/compose/compose_slmm_islmpi_pack.cpp +++ b/components/homme/src/share/compose/compose_slmm_islmpi_pack.cpp @@ -1,46 +1,14 @@ #include "compose_slmm_islmpi.hpp" +#include "compose_slmm_islmpi_buf.hpp" namespace homme { namespace islmpi { -template SLMM_KIF -Int setbuf (Buffer& buf, const Int& os, const Int& i1, const Int& i2) { - Int* const b = reinterpret_cast(&buf(os)); - b[0] = i1; - b[1] = i2; - return nreal_per_2int; -} - -template SLMM_KIF -Int setbuf (Buffer& buf, const Int& os, const Int& i1, const short& i2, const short& i3) { - static_assert(sizeof(Int) >= 2*sizeof(short), "Need >= 2 shorts per Int"); - Int* const b = reinterpret_cast(&buf(os)); - b[0] = i1; - short* const b2 = reinterpret_cast(b+1); - b2[0] = i2; - b2[1] = i3; - return nreal_per_2int; -} - -template SLMM_KIF -Int setbuf (Buffer& buf, const Int& os, const Int& i1, const Int& i2, - const bool final) { - if (final) setbuf(buf, os, i1, i2); - return nreal_per_2int; -} - -template SLMM_KIF -Int setbuf (Buffer& buf, const Int& os, const Int& i1, const short& i2, const short& i3, - const bool final) { - if (final) setbuf(buf, os, i1, i2, i3); - return nreal_per_2int; -} - #ifdef COMPOSE_PORT /* GPU metadata are arranged differently than described below. The scheme is the following: - (#x-in-rank int - x-bulk-data-offset i + (x-bulk-data-offset int + #x-in-rank i (lid-on-rank i only packed if #x in lid > 0 lev short #x) s @@ -56,7 +24,7 @@ struct Accum { }; template -void pack_dep_points_sendbuf_pass1_scan (IslMpi& cm) { +void pack_dep_points_sendbuf_pass1_scan (IslMpi& cm, const bool trajectory) { ko::fence(); deep_copy(cm.nx_in_rank_h, cm.nx_in_rank); const auto& sendbufs = cm.sendbuf; @@ -68,6 +36,7 @@ void pack_dep_points_sendbuf_pass1_scan (IslMpi& cm) { const auto& blas = cm.bla; const auto nlev = cm.nlev; const Int nrmtrank = static_cast(cm.ranks.size()) - 1; + const Int ndim = trajectory ? cm.dep_points_ndim : 3; for (Int ri = 0; ri < nrmtrank; ++ri) { const Int lid_on_rank_n = cm.lid_on_rank_h(ri).n(); const auto f = COMPOSE_LAMBDA (const int idx, Accum& a, const bool fin) { @@ -97,10 +66,10 @@ void pack_dep_points_sendbuf_pass1_scan (IslMpi& cm) { if (nx > 0) { const auto dos = setbuf(sendbuf, a.mos, lid_on_rank(lidi), lev, nx, fin); a.mos += dos; - a.sendcount += dos + 3*nx; + a.sendcount += dos + ndim*nx; if (fin) t.xptr = a.xos; - a.xos += 3*nx; - a.qos += 2 + nx; + a.xos += ndim*nx; + a.qos += trajectory ? nx : 2 + nx; } }; Accum a; @@ -121,8 +90,8 @@ void pack_dep_points_sendbuf_pass1_scan (IslMpi& cm) { /* Pack the departure points (x). We use two passes. We also set up the q metadata. Two passes let us do some efficient tricks that are not available with one pass. Departure point and q messages are formatted as follows: - xs: (#x-in-rank int <- - x-bulk-data-offset i | + xs: (x-bulk-data-offset int <- + #x-in-rank i | (lid-on-rank i only packed if #x in lid > 0 | #x-in-lid i > 0 |- meta data (lev i only packed if #x in (lid,lev) > 0 | @@ -135,7 +104,7 @@ void pack_dep_points_sendbuf_pass1_scan (IslMpi& cm) { *#x) *#lev *#lid *#rank */ template -void pack_dep_points_sendbuf_pass1_noscan (IslMpi& cm) { +void pack_dep_points_sendbuf_pass1_noscan (IslMpi& cm, const bool trajectory) { #ifdef COMPOSE_PORT ko::fence(); deep_copy(cm.nx_in_rank_h, cm.nx_in_rank); @@ -143,6 +112,7 @@ void pack_dep_points_sendbuf_pass1_noscan (IslMpi& cm) { deep_copy(cm.bla_h, cm.bla); #endif const Int nrmtrank = static_cast(cm.ranks.size()) - 1; + const Int ndim = trajectory ? cm.dep_points_ndim : 3; #ifdef COMPOSE_HORIZ_OPENMP # pragma omp for #endif @@ -179,10 +149,10 @@ void pack_dep_points_sendbuf_pass1_noscan (IslMpi& cm) { slmm_assert_high(nx > 0); const auto dos = setbuf(sendbuf, mos, lev, nx); mos += dos; - sendcount += dos + 3*nx; + sendcount += dos + ndim*nx; t.xptr = xos; - xos += 3*nx; - qos += 2 + nx; + xos += ndim*nx; + qos += trajectory ? nx : 2 + nx; nx_in_lid -= nx; } slmm_assert(nx_in_lid == 0); @@ -210,17 +180,18 @@ void pack_dep_points_sendbuf_pass1_noscan (IslMpi& cm) { } template -void pack_dep_points_sendbuf_pass1 (IslMpi& cm) { +void pack_dep_points_sendbuf_pass1 (IslMpi& cm, const bool trajectory) { #if defined COMPOSE_PORT && ! defined COMPOSE_PACK_NOSCAN if (ko::OnGpu::value) - pack_dep_points_sendbuf_pass1_scan(cm); + pack_dep_points_sendbuf_pass1_scan(cm, trajectory); else #endif - pack_dep_points_sendbuf_pass1_noscan(cm); + pack_dep_points_sendbuf_pass1_noscan(cm, trajectory); } template -void pack_dep_points_sendbuf_pass2 (IslMpi& cm, const DepPoints& dep_points) { +void pack_dep_points_sendbuf_pass2 (IslMpi& cm, const DepPoints& dep_points, + const bool trajectory) { const auto myrank = cm.p->rank(); #ifdef COMPOSE_PORT const Int start = 0, end = cm.mylid_with_comm_h.n(); @@ -242,6 +213,7 @@ void pack_dep_points_sendbuf_pass2 (IslMpi& cm, const DepPoints& dep_poi } { ConstExceptGnu Int np2 = cm.np2, nlev = cm.nlev, qsize = cm.qsize; + ConstExceptGnu Int ndim = trajectory ? cm.dep_points_ndim : 3; const auto& ed_d = cm.ed_d; const auto& mylid_with_comm_d = cm.mylid_with_comm_d; const auto& sendbuf = cm.sendbuf; @@ -279,17 +251,21 @@ void pack_dep_points_sendbuf_pass2 (IslMpi& cm, const DepPoints& dep_poi ++t.cnt; #endif qptr = t.qptr; - xptr = x_bulkdata_offset(ri) + t.xptr + 3*cnt; + xptr = x_bulkdata_offset(ri) + t.xptr + ndim*cnt; } #ifdef COMPOSE_HORIZ_OPENMP if (horiz_openmp) omp_unset_lock(lock); #endif slmm_kernel_assert_high(xptr > 0); - for (Int i = 0; i < 3; ++i) + for (Int i = 0; i < ndim; ++i) sb(xptr + i) = dep_points(tci,lev,k,i); auto& item = ed.rmt.atomic_inc_and_return_next(); - item.q_extrema_ptr = qsize * qptr; - item.q_ptr = item.q_extrema_ptr + qsize*(2 + cnt); + if (trajectory) { + item.q_extrema_ptr = item.q_ptr = ndim*(qptr + cnt); + } else { + item.q_extrema_ptr = qsize * qptr; + item.q_ptr = item.q_extrema_ptr + qsize*(2 + cnt); + } item.lev = lev; item.k = k; }; @@ -300,9 +276,11 @@ void pack_dep_points_sendbuf_pass2 (IslMpi& cm, const DepPoints& dep_poi } } -template void pack_dep_points_sendbuf_pass1(IslMpi& cm); -template void pack_dep_points_sendbuf_pass2(IslMpi& cm, - const DepPoints& dep_points); +template void pack_dep_points_sendbuf_pass1( + IslMpi& cm, const bool trajectory); +template void pack_dep_points_sendbuf_pass2( + IslMpi& cm, const DepPoints& dep_points, + const bool trajectory); } // namespace islmpi } // namespace homme diff --git a/components/homme/src/share/compose/compose_slmm_islmpi_q.cpp b/components/homme/src/share/compose/compose_slmm_islmpi_q.cpp index c6633a0f5aeb..aa848de4138a 100644 --- a/components/homme/src/share/compose/compose_slmm_islmpi_q.cpp +++ b/components/homme/src/share/compose/compose_slmm_islmpi_q.cpp @@ -1,198 +1,10 @@ #include "compose_slmm_islmpi.hpp" - -namespace slmm { -static Int test_gll () { - Int nerr = 0; - const Real tol = 1e2*std::numeric_limits::epsilon(); - GLL gll; - const Real* x, * wt; - for (Int np = 2; np <= 4; ++np) { - for (Int monotone_type = 0; monotone_type <= 1; ++monotone_type) { - const Basis b(np, monotone_type); - gll.get_coef(b, x, wt); - Real sum = 0; - for (Int i = 0; i < b.np; ++i) - sum += wt[i]; - if (std::abs(2 - sum) > tol) { - std::cerr << "test_gll " << np << ", " << monotone_type - << ": 2 - sum = " << 2 - sum << "\n"; - ++nerr; - } - for (Int j = 0; j < b.np; ++j) { - Real gj[GLL::np_max]; - gll.eval(b, x[j], gj); - for (Int i = 0; i < b.np; ++i) { - if (j == i) continue; - if (std::abs(gj[i]) > tol) { - std::cerr << "test_gll " << np << ", " << monotone_type << ": gj[" - << i << "] = " << gj[i] << "\n"; - ++nerr; - } - } - } - } - } - for (Int np = 2; np <= 4; ++np) { - const Basis b(np, 0); - Real a[] = {-0.9, -0.7, -0.3, 0.1, 0.2, 0.4, 0.6, 0.8}; - const Real delta = std::sqrt(std::numeric_limits::epsilon()); - for (size_t ia = 0; ia < sizeof(a)/sizeof(Real); ++ia) { - Real gj[GLL::np_max], gjp[GLL::np_max], gjm[GLL::np_max]; - gll.eval_derivative(b, a[ia], gj); - gll.eval(b, a[ia] + delta, gjp); - gll.eval(b, a[ia] - delta, gjm); - for (Int i = 0; i < b.np; ++i) { - const Real fd = (gjp[i] - gjm[i])/(2*delta); - if (std::abs(fd - gj[i]) >= delta*std::abs(gjp[i])) - ++nerr; - } - } - } - return nerr; -} - -int unittest () { - int nerr = 0; - nerr += test_gll(); - return nerr; -} - -static constexpr Real sqrt5 = 2.23606797749978969641; // std::sqrt(5.0); -static constexpr Real oosqrt5 = 1.0 / sqrt5; - -SLMM_KF void gll_np4_eval (const Real x, Real y[4]) { - static constexpr Real oo8 = 1.0/8.0; - const Real x2 = x*x; - y[0] = (1.0 - x)*(5.0*x2 - 1.0)*oo8; - y[1] = -sqrt5*oo8*(sqrt5 - 5.0*x)*(x2 - 1.0); - y[2] = -sqrt5*oo8*(sqrt5 + 5.0*x)*(x2 - 1.0); - y[3] = (1.0 + x)*(5.0*x2 - 1.0)*oo8; -} - -// Linear interp in each region. -SLMM_KF void gll_np4_subgrid_eval_impl (const Real& x, Real y[4]) { - if (x < -oosqrt5) { - const Real alpha = (x + 1)/(1 - oosqrt5); - y[0] = 1 - alpha; - y[1] = alpha; - y[2] = 0; - y[3] = 0; - } else { - const Real alpha = (x + oosqrt5)/(2*oosqrt5); - y[0] = 0; - y[1] = 1 - alpha; - y[2] = alpha; - y[3] = 0; - } -} - -SLMM_KF void gll_np4_subgrid_eval (const Real& x, Real y[4]) { - if (x > 0) { - gll_np4_subgrid_eval_impl(-x, y); - ko::swap(y[0], y[3]); - ko::swap(y[1], y[2]); - return; - } - gll_np4_subgrid_eval_impl(x, y); -} - -// Quadratic interpolant across nodes 1,2,3 -- i.e., excluding node 0 -- of the -// np=4 reference element. -SLMM_KF void outer_eval (const Real& x, Real v[4]) { - static constexpr Real - xbar = (2*oosqrt5) / (1 + oosqrt5), - ooxbar = 1 / xbar, - ybar = 1 / (xbar - 1); - const Real xn = (x + oosqrt5) / (1 + oosqrt5); - v[0] = 0; - v[1] = 1 + ybar*xn*((1 - ooxbar)*xn + ooxbar - xbar); - v[2] = ybar*ooxbar*xn*(xn - 1); - v[3] = ybar*xn*(xbar - xn); -} - -// In the middle region, use the standard GLL np=4 interpolant; in the two outer -// regions, use an order-reduced interpolant that stabilizes the method. -SLMM_KF void gll_np4_subgrid_exp_eval (const Real& x, Real y[4]) { - static constexpr Real - alpha = 0.5527864045000416708, - v = 0.427*(1 + alpha), - x2 = 0.4472135954999579277, - x3 = 1 - x2, - det = x2*x3*(x2 - x3), - y2 = alpha, - y3 = v, - c1 = (x3*y2 - x2*y3)/det, - c2 = (-x3*x3*y2 + x2*x2*y3)/det; - if (x < -oosqrt5 || x > oosqrt5) { - if (x < -oosqrt5) { - outer_eval(-x, y); - ko::swap(y[0], y[3]); - ko::swap(y[1], y[2]); - } else - outer_eval(x, y); - Real y4[4]; - gll_np4_eval(x, y4); - const Real x0 = 1 - std::abs(x); - const Real a = (c1*x0 + c2)*x0; - for (int i = 0; i < 4; ++i) - y[i] = a*y[i] + (1 - a)*y4[i]; - } else - gll_np4_eval(x, y); -} -} // namespace slmm +#include "compose_slmm_islmpi_interpolate.hpp" +#include "compose_slmm_islmpi_buf.hpp" namespace homme { namespace islmpi { -template -SLMM_KIF void interpolate (const typename IslMpi::Advecter::Alg::Enum& alg, - const Real ref_coord[2], Real rx[4], Real ry[4]) { - typedef typename IslMpi::Advecter::Alg Alg; - switch (alg) { - case Alg::csl_gll: - slmm::gll_np4_eval(ref_coord[0], rx); - slmm::gll_np4_eval(ref_coord[1], ry); - break; - case Alg::csl_gll_subgrid: - slmm::gll_np4_subgrid_eval(ref_coord[0], rx); - slmm::gll_np4_subgrid_eval(ref_coord[1], ry); - break; - case Alg::csl_gll_exp: - slmm::gll_np4_subgrid_exp_eval(ref_coord[0], rx); - slmm::gll_np4_subgrid_exp_eval(ref_coord[1], ry); - break; - default: - slmm_kernel_assert(0); - } -} - -SLMM_KIF Real calc_q_tgt (const Real rx[4], const Real ry[4], const Real qs[16]) { - return (ry[0]*(rx[0]*qs[ 0] + rx[1]*qs[ 1] + rx[2]*qs[ 2] + rx[3]*qs[ 3]) + - ry[1]*(rx[0]*qs[ 4] + rx[1]*qs[ 5] + rx[2]*qs[ 6] + rx[3]*qs[ 7]) + - ry[2]*(rx[0]*qs[ 8] + rx[1]*qs[ 9] + rx[2]*qs[10] + rx[3]*qs[11]) + - ry[3]*(rx[0]*qs[12] + rx[1]*qs[13] + rx[2]*qs[14] + rx[3]*qs[15])); -} - -SLMM_KIF Real calc_q_tgt (const Real rx[4], const Real ry[4], const Real qdp[16], - const Real dp[16]) { - return (ry[0]*(rx[0]*(qdp[ 0]/dp[ 0]) + rx[1]*(qdp[ 1]/dp[ 1]) + - rx[2]*(qdp[ 2]/dp[ 2]) + rx[3]*(qdp[ 3]/dp[ 3])) + - ry[1]*(rx[0]*(qdp[ 4]/dp[ 4]) + rx[1]*(qdp[ 5]/dp[ 5]) + - rx[2]*(qdp[ 6]/dp[ 6]) + rx[3]*(qdp[ 7]/dp[ 7])) + - ry[2]*(rx[0]*(qdp[ 8]/dp[ 8]) + rx[1]*(qdp[ 9]/dp[ 9]) + - rx[2]*(qdp[10]/dp[10]) + rx[3]*(qdp[11]/dp[11])) + - ry[3]*(rx[0]*(qdp[12]/dp[12]) + rx[1]*(qdp[13]/dp[13]) + - rx[2]*(qdp[14]/dp[14]) + rx[3]*(qdp[15]/dp[15]))); -} - -template SLMM_KIF -Int getbuf (Buffer& buf, const Int& os, Int& i1, Int& i2) { - const Int* const b = reinterpret_cast(&buf(os)); - i1 = b[0]; - i2 = b[1]; - return nreal_per_2int; -} - #ifndef COMPOSE_PORT // Homme computational pattern. @@ -267,7 +79,7 @@ void calc_own_q (IslMpi& cm, const Int& nets, const Int& nete, auto& ed = cm.ed_d(tci); const FA3 q_tgt(ed.q, cm.np2, cm.nlev, cm.qsize); const Int ned = ed.own.n(); -#ifdef HORIZ_OPENMP +#ifdef COMPOSE_HORIZ_OPENMP # pragma omp for #endif for (Int idx = 0; idx < ned; ++idx) { @@ -317,7 +129,7 @@ template void calc_rmt_q_pass2 (IslMpi& cm) { const Int qsize = cm.qsize; -#ifdef HORIZ_OPENMP +#ifdef COMPOSE_HORIZ_OPENMP # pragma omp for #endif for (Int it = 0; it < cm.nrmt_qs_extrema; ++it) { @@ -331,7 +143,7 @@ void calc_rmt_q_pass2 (IslMpi& cm) { qs(qos + 2*iq + i) = ed.q_extrema(iq, lev, i); } -#ifdef HORIZ_OPENMP +#ifdef COMPOSE_HORIZ_OPENMP # pragma omp for #endif for (Int it = 0; it < cm.nrmt_xs; ++it) { @@ -466,12 +278,13 @@ struct Accum { } }; -template -void calc_rmt_q_pass1_scan (IslMpi& cm) { +template +void calc_rmt_q_pass1_scan (IslMpi& cm, const bool trajectory) { const auto& recvbuf = cm.recvbuf; const auto& rmt_xs = cm.rmt_xs; const auto& rmt_qs_extrema = cm.rmt_qs_extrema; const Int nrmtrank = static_cast(cm.ranks.size()) - 1; + const Int ndim = trajectory ? cm.dep_points_ndim : 3; Int cnt = 0, qcnt = 0; for (Int ri = 0; ri < nrmtrank; ++ri) { const auto get_xos = COMPOSE_LAMBDA (const Int, Int& xos) { @@ -492,7 +305,7 @@ void calc_rmt_q_pass1_scan (IslMpi& cm) { short lev, nx; getbuf(xs, (idx + 1)*nreal_per_2int, lid, lev, nx); slmm_kernel_assert(nx > 0); - if (fin) { + if (fin && ! trajectory) { const auto qcnt_tot = qcnt + a.qcnt; rmt_qs_extrema(4*qcnt_tot + 0) = ri; rmt_qs_extrema(4*qcnt_tot + 1) = lid; @@ -500,7 +313,8 @@ void calc_rmt_q_pass1_scan (IslMpi& cm) { rmt_qs_extrema(4*qcnt_tot + 3) = a.qos; } a.qcnt += 1; - a.qos += 2; + if ( ! trajectory) + a.qos += 2; if (fin) { for (Int xi = 0; xi < nx; ++xi) { const auto cnt_tot = cnt + a.cnt; @@ -510,23 +324,23 @@ void calc_rmt_q_pass1_scan (IslMpi& cm) { rmt_xs(5*cnt_tot + 3) = xos + a.xos; rmt_xs(5*cnt_tot + 4) = a.qos; a.cnt += 1; - a.xos += 3; + a.xos += ndim; a.qos += 1; } } else { a.cnt += nx; - a.xos += 3*nx; - a.qos += nx; + a.xos += ndim*nx; + a.qos += nx; } }; Accum a; ko::parallel_scan(ko::RangePolicy(0, xos/nreal_per_2int - 1), f, a); - cm.sendcount_h(ri) = cm.qsize*a.qos; + cm.sendcount_h(ri) = (trajectory ? ndim : cm.qsize)*a.qos; cnt += a.cnt; qcnt += a.qcnt; } cm.nrmt_xs = cnt; - cm.nrmt_qs_extrema = qcnt; + cm.nrmt_qs_extrema = trajectory ? 0 : qcnt; } template @@ -593,13 +407,20 @@ void calc_rmt_q_pass2 (IslMpi& cm) { #endif // COMPOSE_PORT -template -void calc_rmt_q_pass1_noscan (IslMpi& cm) { +template +void calc_rmt_q_pass1_noscan (IslMpi& cm, const bool trajectory) { const Int nrmtrank = static_cast(cm.ranks.size()) - 1; + const Int ndim = trajectory ? cm.dep_points_ndim : 3; #ifdef COMPOSE_PORT_SEPARATE_VIEWS +#ifdef COMPOSE_HORIZ_OPENMP +# pragma omp for +#endif for (Int ri = 0; ri < nrmtrank; ++ri) ko::deep_copy(ko::View(cm.recvbuf_meta_h(ri).data(), 1), ko::View(cm.recvbuf.get_h(ri).data(), 1)); +#ifdef COMPOSE_HORIZ_OPENMP +# pragma omp for +#endif for (Int ri = 0; ri < nrmtrank; ++ri) { const auto&& xs = cm.recvbuf_meta_h(ri); Int n, unused; @@ -610,71 +431,79 @@ void calc_rmt_q_pass1_noscan (IslMpi& cm) { ko::View(cm.recvbuf.get_h(ri).data(), n)); } #endif - Int cnt = 0, qcnt = 0; - for (Int ri = 0; ri < nrmtrank; ++ri) { - const auto&& xs = cm.recvbuf_meta_h(ri); - Int mos = 0, qos = 0, nx_in_rank, xos; - mos += getbuf(xs, mos, xos, nx_in_rank); - if (nx_in_rank == 0) { - cm.sendcount_h(ri) = 0; - continue; - } - // The upper bound is to prevent an inf loop if the msg is corrupted. - for (Int lidi = 0; lidi < cm.nelemd; ++lidi) { - Int lid, nx_in_lid; - mos += getbuf(xs, mos, lid, nx_in_lid); - for (Int levi = 0; levi < cm.nlev; ++levi) { // same re: inf loop - Int lev, nx; - mos += getbuf(xs, mos, lev, nx); - slmm_assert(nx > 0); - { - cm.rmt_qs_extrema_h(4*qcnt + 0) = ri; - cm.rmt_qs_extrema_h(4*qcnt + 1) = lid; - cm.rmt_qs_extrema_h(4*qcnt + 2) = lev; - cm.rmt_qs_extrema_h(4*qcnt + 3) = qos; - ++qcnt; - qos += 2; - } - for (Int xi = 0; xi < nx; ++xi) { - cm.rmt_xs_h(5*cnt + 0) = ri; - cm.rmt_xs_h(5*cnt + 1) = lid; - cm.rmt_xs_h(5*cnt + 2) = lev; - cm.rmt_xs_h(5*cnt + 3) = xos; - cm.rmt_xs_h(5*cnt + 4) = qos; - ++cnt; - xos += 3; - ++qos; +#ifdef COMPOSE_HORIZ_OPENMP +# pragma omp master +#endif + { + Int cnt = 0, qcnt = 0; + for (Int ri = 0; ri < nrmtrank; ++ri) { + const auto&& xs = cm.recvbuf_meta_h(ri); + Int mos = 0, qos = 0, nx_in_rank, xos; + mos += getbuf(xs, mos, xos, nx_in_rank); + if (nx_in_rank == 0) { + cm.sendcount_h(ri) = 0; + continue; + } + // The upper bound is to prevent an inf loop if the msg is corrupted. + for (Int lidi = 0; lidi < cm.nelemd; ++lidi) { + Int lid, nx_in_lid; + mos += getbuf(xs, mos, lid, nx_in_lid); + for (Int levi = 0; levi < cm.nlev; ++levi) { // same re: inf loop + Int lev, nx; + mos += getbuf(xs, mos, lev, nx); + slmm_assert(nx > 0); + if ( ! trajectory) { + cm.rmt_qs_extrema_h(4*qcnt + 0) = ri; + cm.rmt_qs_extrema_h(4*qcnt + 1) = lid; + cm.rmt_qs_extrema_h(4*qcnt + 2) = lev; + cm.rmt_qs_extrema_h(4*qcnt + 3) = qos; + ++qcnt; + qos += 2; + } + for (Int xi = 0; xi < nx; ++xi) { + cm.rmt_xs_h(5*cnt + 0) = ri; + cm.rmt_xs_h(5*cnt + 1) = lid; + cm.rmt_xs_h(5*cnt + 2) = lev; + cm.rmt_xs_h(5*cnt + 3) = xos; + cm.rmt_xs_h(5*cnt + 4) = qos; + ++cnt; + xos += ndim; + ++qos; + } + nx_in_lid -= nx; + nx_in_rank -= nx; + if (nx_in_lid == 0) break; } - nx_in_lid -= nx; - nx_in_rank -= nx; - if (nx_in_lid == 0) break; + slmm_assert(nx_in_lid == 0); + if (nx_in_rank == 0) break; } - slmm_assert(nx_in_lid == 0); - if (nx_in_rank == 0) break; + slmm_assert(nx_in_rank == 0); + cm.sendcount_h(ri) = (trajectory ? ndim : cm.qsize)*qos; } - slmm_assert(nx_in_rank == 0); - cm.sendcount_h(ri) = cm.qsize*qos; + cm.nrmt_xs = cnt; + cm.nrmt_qs_extrema = trajectory ? 0 : qcnt; + deep_copy(cm.rmt_xs, cm.rmt_xs_h); + deep_copy(cm.rmt_qs_extrema, cm.rmt_qs_extrema_h); } - cm.nrmt_xs = cnt; - cm.nrmt_qs_extrema = qcnt; - deep_copy(cm.rmt_xs, cm.rmt_xs_h); - deep_copy(cm.rmt_qs_extrema, cm.rmt_qs_extrema_h); +#ifdef COMPOSE_HORIZ_OPENMP +# pragma omp barrier +#endif } -template -void calc_rmt_q_pass1 (IslMpi& cm) { +template +void calc_rmt_q_pass1 (IslMpi& cm, const bool trajectory) { #if defined COMPOSE_PORT && ! defined COMPOSE_PACK_NOSCAN if (ko::OnGpu::value) - calc_rmt_q_pass1_scan(cm); + calc_rmt_q_pass1_scan(cm, trajectory); else #endif - calc_rmt_q_pass1_noscan(cm); + calc_rmt_q_pass1_noscan(cm, trajectory); } template void calc_rmt_q (IslMpi& cm) { { slmm::Timer t("09_rmt_q_pass1"); - calc_rmt_q_pass1(cm); } + calc_rmt_q_pass1(cm); } { slmm::Timer t("09_rmt_q_pass2"); calc_rmt_q_pass2(cm); } } @@ -697,6 +526,8 @@ void calc_rmt_q (IslMpi& cm) { } } +template void calc_rmt_q_pass1(IslMpi& cm, + const bool trajectory); template void calc_rmt_q(IslMpi& cm); template void calc_own_q(IslMpi& cm, const Int& nets, const Int& nete, diff --git a/components/homme/src/share/compose/compose_slmm_islmpi_step.cpp b/components/homme/src/share/compose/compose_slmm_islmpi_step.cpp index 1ae91c3e5382..3c707c8919fa 100644 --- a/components/homme/src/share/compose/compose_slmm_islmpi_step.cpp +++ b/components/homme/src/share/compose/compose_slmm_islmpi_step.cpp @@ -24,11 +24,13 @@ void step ( const auto& q_min = cm.tracer_arrays->q_min; const auto& q_max = cm.tracer_arrays->q_max; #else - const DepPointsH dep_points(dep_points_r, cm.nelemd, cm.nlev, cm.np2); + const DepPointsH dep_points(dep_points_r, cm.nelemd, cm.nlev, cm.np2, + cm.dep_points_ndim); const QExtremaH q_min(q_min_r, cm.nelemd, cm.qsize, cm.nlev, cm.np2), q_max(q_max_r, cm.nelemd, cm.qsize, cm.nlev, cm.np2); #endif + slmm_assert(dep_points.extent_int(3) == cm.dep_points_ndim); // Partition my elements that communicate with remotes among threads, if I // haven't done that yet. diff --git a/components/homme/src/share/compose/compose_test.cpp b/components/homme/src/share/compose/compose_test.cpp index 4b29f3ae7c32..944114d79758 100644 --- a/components/homme/src/share/compose/compose_test.cpp +++ b/components/homme/src/share/compose/compose_test.cpp @@ -327,7 +327,7 @@ struct StandaloneTracersTester { #endif } }; - + static StandaloneTracersTester::Ptr g_stt; } // namespace test } // namespace compose diff --git a/components/homme/src/share/compose/compose_test.hpp b/components/homme/src/share/compose/compose_test.hpp index 9a3e7d4b350b..ff4d18cdfd9c 100644 --- a/components/homme/src/share/compose/compose_test.hpp +++ b/components/homme/src/share/compose/compose_test.hpp @@ -13,6 +13,7 @@ namespace test { int slmm_unittest(); int cedr_unittest(); int cedr_unittest(MPI_Comm comm); +int interpolate_unittest(); typedef double Real; typedef int Int; diff --git a/components/homme/src/share/compose_mod.F90 b/components/homme/src/share/compose_mod.F90 index 8b134def384b..6085eaf1238d 100644 --- a/components/homme/src/share/compose_mod.F90 +++ b/components/homme/src/share/compose_mod.F90 @@ -25,22 +25,24 @@ subroutine cedr_unittest(comm, nerr) bind(c) end subroutine cedr_unittest subroutine cedr_init_impl(comm, cdr_alg, use_sgi, gid_data, rank_data, & - ncell, nlclcell, nlev, qsize, independent_time_steps, hard_zero, & + ncell, nlclcell, nlev, np, qsize, independent_time_steps, hard_zero, & gid_data_sz, rank_data_sz) bind(c) use iso_c_binding, only: c_int, c_bool - integer(kind=c_int), value, intent(in) :: comm, cdr_alg, ncell, nlclcell, nlev, & + integer(kind=c_int), value, intent(in) :: comm, cdr_alg, ncell, nlclcell, nlev, np, & qsize, gid_data_sz, rank_data_sz logical(kind=c_bool), value, intent(in) :: use_sgi, independent_time_steps, hard_zero integer(kind=c_int), intent(in) :: gid_data(gid_data_sz), rank_data(rank_data_sz) end subroutine cedr_init_impl subroutine slmm_init_impl(comm, transport_alg, np, nlev, qsize, qsize_d, & - nelem, nelemd, cubed_sphere_map, geometry, lid2gid, lid2facenum, nbr_id_rank, nirptr, & - sl_nearest_point_lev, lid2gid_sz, lid2facenum_sz, nbr_id_rank_sz, nirptr_sz) bind(c) + nelem, nelemd, cubed_sphere_map, geometry, lid2gid, lid2facenum, & + nbr_id_rank, nirptr, sl_halo, sl_traj_3d, sl_traj_nsubstep, sl_nearest_point_lev, & + lid2gid_sz, lid2facenum_sz, nbr_id_rank_sz, nirptr_sz) bind(c) use iso_c_binding, only: c_int - integer(kind=c_int), value, intent(in) :: comm, transport_alg, np, nlev, qsize, qsize_d, & - nelem, nelemd, cubed_sphere_map, geometry, sl_nearest_point_lev, lid2gid_sz, & - lid2facenum_sz, nbr_id_rank_sz, nirptr_sz + integer(kind=c_int), value, intent(in) :: comm, transport_alg, np, nlev, qsize, & + qsize_d, nelem, nelemd, cubed_sphere_map, geometry, sl_halo, sl_traj_3d, & + sl_traj_nsubstep, sl_nearest_point_lev, lid2gid_sz, lid2facenum_sz, & + nbr_id_rank_sz, nirptr_sz integer(kind=c_int), intent(in) :: lid2gid(lid2gid_sz), lid2facenum(lid2facenum_sz), & nbr_id_rank(nbr_id_rank_sz), nirptr(nirptr_sz) end subroutine slmm_init_impl @@ -186,7 +188,28 @@ subroutine slmm_check_ref2sphere(ie, sphere_cart_coord) bind(c) type(cartesian3D_t), intent(in) :: sphere_cart_coord end subroutine slmm_check_ref2sphere - subroutine slmm_csl_set_elem_data(ie, metdet, qdp, n0_qdp, dp, q, nelem_in_patch, h2d, d2h) bind(c) + subroutine slmm_set_hvcoord(etai_beg, etai_end, etam) bind(c) + use iso_c_binding, only: c_double + use dimensions_mod, only : nlev + real(kind=c_double), value, intent(in) :: etai_beg, etai_end + real(kind=c_double), intent(in) :: etam(nlev) + end subroutine slmm_set_hvcoord + + subroutine slmm_calc_v_departure(nets, nete, step, dtsub, dep_points, & + dep_points_ndim, vnode, vdep, info) bind(c) + use iso_c_binding, only: c_int, c_double + use dimensions_mod, only : np, nlev, nelemd, qsize + use coordinate_systems_mod, only : cartesian3D_t + integer(kind=c_int), value, intent(in) :: nets, nete, step, dep_points_ndim + real(kind=c_double), value, intent(in) :: dtsub + real(kind=c_double), intent(inout) :: dep_points(dep_points_ndim,np,np,nlev,nelemd) + real(kind=c_double), intent(in) :: vnode(dep_points_ndim,np,np,nlev,nelemd) + real(kind=c_double), intent(out) :: vdep(dep_points_ndim,np,np,nlev,nelemd) + integer(kind=c_int), intent(out) :: info + end subroutine slmm_calc_v_departure + + subroutine slmm_csl_set_elem_data(ie, metdet, qdp, n0_qdp, dp, q, nelem_in_patch, & + h2d, d2h) bind(c) use iso_c_binding, only: c_int, c_double, c_bool use dimensions_mod, only : nlev, np, qsize real(kind=c_double), intent(in) :: metdet(np,np), qdp(np,np,nlev,qsize,2), & @@ -195,17 +218,17 @@ subroutine slmm_csl_set_elem_data(ie, metdet, qdp, n0_qdp, dp, q, nelem_in_patch logical(kind=c_bool), value, intent(in) :: h2d, d2h end subroutine slmm_csl_set_elem_data - subroutine slmm_csl(nets, nete, dep_points, minq, maxq, info) bind(c) + subroutine slmm_csl(nets, nete, dep_points, dep_points_ndim, minq, maxq, info) bind(c) use iso_c_binding, only: c_int, c_double use dimensions_mod, only : np, nlev, nelemd, qsize use coordinate_systems_mod, only : cartesian3D_t - integer(kind=c_int), value, intent(in) :: nets, nete + integer(kind=c_int), value, intent(in) :: nets, nete, dep_points_ndim ! dep_points is const in principle, but if lev <= ! semi_lagrange_nearest_point_lev, a departure point may be altered if ! the winds take it outside of the comm halo. - type(cartesian3D_t), intent(inout) :: dep_points(np,np,nlev,nelemd) + real(kind=c_double), intent(inout) :: dep_points(dep_points_ndim,np,np,nlev,nelemd) real(kind=c_double), intent(in) :: & - minq(np,np,nlev,qsize,nets:nete), maxq(np,np,nlev,qsize,nets:nete) + minq(np,np,nlev,qsize,nelemd), maxq(np,np,nlev,qsize,nelemd) integer(kind=c_int), intent(out) :: info end subroutine slmm_csl @@ -238,6 +261,7 @@ subroutine compose_init(par, elem, GridVertex, init_kokkos) use element_mod, only: element_t use gridgraph_mod, only: GridVertex_t use control_mod, only: semi_lagrange_cdr_alg, transport_alg, cubed_sphere_map, & + semi_lagrange_halo, semi_lagrange_trajectory_nsubstep, & semi_lagrange_nearest_point_lev, dt_remap_factor, dt_tracer_factor, geometry use physical_constants, only: Sx, Sy, Lx, Ly use scalable_grid_init_mod, only: sgi_is_initialized, sgi_get_rank2sfc, & @@ -254,7 +278,7 @@ subroutine compose_init(par, elem, GridVertex, init_kokkos) ! These are for non-scalable grid initialization, still used for RRM. sc2gci(:), sc2rank(:) ! space curve index -> (GID, rank) integer :: lid2gid(nelemd), lid2facenum(nelemd) - integer :: i, j, k, sfc, gid, igv, sc, geometry_type + integer :: i, j, k, sfc, gid, igv, sc, geometry_type, sl_traj_3d ! To map SFC index to IDs and ranks logical(kind=c_bool) :: use_sgi, owned, independent_time_steps, hard_zero integer, allocatable :: owned_ids(:) @@ -273,6 +297,16 @@ subroutine compose_init(par, elem, GridVertex, init_kokkos) hard_zero = .true. independent_time_steps = dt_remap_factor < dt_tracer_factor + + if (semi_lagrange_halo < 1) then + ! For test problems, the relationship between dt_tracer_factor and halo + ! may not be clear. But for real problems, the advective CFL implies that + ! a parcel can cross a cell in three time steps. Since this is closely + ! related to the dynamics' tstep, dt_tracer_factor is meaningful, + ! implying: + semi_lagrange_halo = (dt_tracer_factor + 2) / 3 + if (semi_lagrange_halo < 1) semi_lagrange_halo = 1 + end if geometry_type = 0 ! sphere if (trim(geometry) == "plane") then @@ -316,12 +350,12 @@ subroutine compose_init(par, elem, GridVertex, init_kokkos) if (use_sgi) then if (.not. allocated(owned_ids)) allocate(owned_ids(1)) call cedr_init_impl(par%comm, semi_lagrange_cdr_alg, & - use_sgi, owned_ids, rank2sfc, nelem, nelemd, nlev, qsize, & + use_sgi, owned_ids, rank2sfc, nelem, nelemd, nlev, np, qsize, & independent_time_steps, hard_zero, size(owned_ids), size(rank2sfc)) else if (.not. allocated(sc2gci)) allocate(sc2gci(1), sc2rank(1)) call cedr_init_impl(par%comm, semi_lagrange_cdr_alg, & - use_sgi, sc2gci, sc2rank, nelem, nelemd, nlev, qsize, & + use_sgi, sc2gci, sc2rank, nelem, nelemd, nlev, np, qsize, & independent_time_steps, hard_zero, size(sc2gci), size(sc2rank)) end if if (allocated(sc2gci)) deallocate(sc2gci, sc2rank) @@ -360,9 +394,12 @@ subroutine compose_init(par, elem, GridVertex, init_kokkos) end do end do nirptr(nelemd+1) = k - 1 + sl_traj_3d = 0 + if (independent_time_steps) sl_traj_3d = 1 call slmm_init_impl(par%comm, transport_alg, np, nlev, qsize, qsize_d, & nelem, nelemd, cubed_sphere_map, geometry_type, lid2gid, lid2facenum, & - nbr_id_rank, nirptr, semi_lagrange_nearest_point_lev, & + nbr_id_rank, nirptr, semi_lagrange_halo, sl_traj_3d, & + semi_lagrange_trajectory_nsubstep, semi_lagrange_nearest_point_lev, & size(lid2gid), size(lid2facenum), size(nbr_id_rank), size(nirptr)) if (geometry_type == 1) call slmm_init_plane(Sx, Sy, Lx, Ly) deallocate(nbr_id_rank, nirptr) diff --git a/components/homme/src/share/compose_test_mod.F90 b/components/homme/src/share/compose_test_mod.F90 index 8421e41e5faa..8ba3b7f44d74 100644 --- a/components/homme/src/share/compose_test_mod.F90 +++ b/components/homme/src/share/compose_test_mod.F90 @@ -117,7 +117,7 @@ subroutine compose_test(par, hvcoord, dom_mt, elem, eval) ! 1. Unit tests. call compose_unittest() - call sl_unittest(par) + call sl_unittest(par, hvcoord) nerr = 0 call cedr_unittest(par%comm, nerr) if (nerr /= 0) print *, 'cedr_unittest returned', nerr @@ -240,7 +240,7 @@ subroutine compose_stt(hybrid, dom_mt, nets, nete, hvcoord, deriv, elem, eval) ! nsteps = nint(6*ne*(15.d0/qsplit)) nsteps = nmax if (hybrid%par%masterproc .and. hybrid%masterthread) then - print *, 'COMPOSE> nsteps', nsteps + print '(a,i6,a,i5)', 'COMPOSE> nsteps ', nsteps, ' ne ', ne end if dt = twelve_days / nsteps call t_barrierf('compose_stt_step_start_barrier', hybrid%par%comm) diff --git a/components/homme/src/share/control_mod.F90 b/components/homme/src/share/control_mod.F90 index 0e9494f5a6cd..0e0276fbf2f5 100644 --- a/components/homme/src/share/control_mod.F90 +++ b/components/homme/src/share/control_mod.F90 @@ -26,6 +26,8 @@ module control_mod ! 3 CAAS ! 20 QLT with superlevels ! 30 CAAS with superlevels + ! 4* reserved for debugging + ! 5 CAAS-point integer, public :: semi_lagrange_cdr_alg = 3 ! If true, check mass conservation and shape preservation. The second ! implicitly checks tracer consistency. @@ -39,6 +41,10 @@ module control_mod ! halo available to it if the actual point is outside the halo. This is done ! in levels <= this parameter. integer, public :: semi_lagrange_nearest_point_lev = 256 + integer, public :: semi_lagrange_halo = -1 + integer, public :: semi_lagrange_trajectory_nsubstep = 0 + integer, public :: semi_lagrange_trajectory_nvelocity = -1 + integer, public :: semi_lagrange_diagnostics = 0 ! flag used by preqx, theta-l and theta-c models ! should be renamed to "hydrostatic_mode" diff --git a/components/homme/src/share/cxx/ComposeTransport.cpp b/components/homme/src/share/cxx/ComposeTransport.cpp index 760a3dd06fa4..8d7349093282 100644 --- a/components/homme/src/share/cxx/ComposeTransport.cpp +++ b/components/homme/src/share/cxx/ComposeTransport.cpp @@ -69,9 +69,13 @@ std::vector > ComposeTransport::run_unit_tests () { assert(is_setup); std::vector > fails; - int nerr; - nerr = m_compose_impl->run_trajectory_unit_tests(); - if (nerr) fails.push_back(std::make_pair("run_trajectory_unit_tests", nerr)); + int ne, nerr = 0; + ne = m_compose_impl->run_trajectory_unit_tests(); + if (ne) fails.push_back(std::make_pair("run_trajectory_unit_tests", nerr)); + nerr += ne; + ne = m_compose_impl->run_enhanced_trajectory_unit_tests(); + if (ne) fails.push_back(std::make_pair("run_enhanced_trajectory_unit_tests", nerr)); + nerr += ne; return fails; } diff --git a/components/homme/src/share/cxx/ComposeTransportImpl.hpp b/components/homme/src/share/cxx/ComposeTransportImpl.hpp index 09bd43d9d539..b536b8ba65a7 100644 --- a/components/homme/src/share/cxx/ComposeTransportImpl.hpp +++ b/components/homme/src/share/cxx/ComposeTransportImpl.hpp @@ -41,7 +41,6 @@ struct ComposeTransportImpl { enum : int { max_num_lev_pack = NUM_LEV_P }; enum : int { max_num_lev_aligned = max_num_lev_pack*packn }; enum : int { num_phys_lev = NUM_PHYSICAL_LEV }; - enum : int { num_work = 12 }; static_assert(max_num_lev_aligned >= 3, "We use wrk(0:2,:) and so need max_num_lev_aligned >= 3"); @@ -49,21 +48,56 @@ struct ComposeTransportImpl { using TeamPolicy = Kokkos::TeamPolicy; using MT = typename TeamPolicy::member_type; - using Buf1 = ExecViewUnmanaged; + // For the enhanced trajectory, we need one extra level beyond the usual. + using Buf1Alloc = ExecViewUnmanaged; + using Buf1o = ExecViewUnmanaged; + using Buf1e = Buf1Alloc; + using Buf2 = ExecViewUnmanaged; - using DeparturePoints = ExecViewManaged; + using DeparturePoints = ExecViewManaged; + + typedef ExecViewUnmanaged SNlev; + typedef ExecViewUnmanaged RNlev; + typedef ExecViewUnmanaged SNlevp; + typedef ExecViewUnmanaged RNlevp; + typedef ExecViewUnmanaged S2Nlev; + typedef ExecViewUnmanaged R2Nlev; + typedef ExecViewUnmanaged S2Nlevp; + typedef typename ViewConst::type CSNlev; + typedef typename ViewConst::type CRNlev; + typedef typename ViewConst::type CSNlevp; + typedef typename ViewConst::type CRNlevp; + typedef typename ViewConst::type CS2Nlev; + typedef typename ViewConst::type CR2Nlev; + + using DpSlot = ExecViewUnmanaged< Scalar** [NP][NP][NUM_LEV]>; + using VSlot = ExecViewUnmanaged< Scalar**[2][NP][NP][NUM_LEV]>; + using CDpSlot = ExecViewUnmanaged; + using CVSlot = ExecViewUnmanaged; + struct VelocityRecord; struct Data { int nelemd, qsize, limiter_option, cdr_check, hv_q, hv_subcycle_q; int geometry_type; // 0: sphere, 1: plane - Real nu_q, hv_scaling, dp_tol; + int trajectory_nsubstep; // 0: original alg, >= 1: enhanced + Real nu_q, hv_scaling, dp_tol, deta_tol; bool independent_time_steps; - Buf1 buf1[3]; - Buf2 buf2[2]; + // buf1o and buf1e point to the same memory, sized to the larger of the + // two. They are used in different parts of the code. + static constexpr int n_buf1 = 4, n_buf2 = 4; + Buf1o buf1o[n_buf1]; + Buf1e buf1e[n_buf1]; + Buf2 buf2[n_buf2]; + + ExecView hydetai; // diff(etai) + ExecView hydetam_ref; + + DeparturePoints dep_pts, vnode, vdep; // (ie,lev,i,j,d) - DeparturePoints dep_pts; + std::shared_ptr vrec; Data () : nelemd(-1), qsize(-1), limiter_option(9), cdr_check(0), hv_q(0), @@ -99,6 +133,7 @@ struct ComposeTransportImpl { } void set_dp_tol(); + void setup_enhanced_trajectory(); void reset(const SimulationParams& params); int requested_buffer_size() const; void init_buffers(const FunctorsBuffersManager& fbm); @@ -108,6 +143,7 @@ struct ComposeTransportImpl { void remap_q(const TimeLevel& tl); void calc_trajectory(const int np1, const Real dt); + void calc_enhanced_trajectory(const int np1, const Real dt); void remap_v(const ExecViewUnmanaged& dp3d, const int np1, const ExecViewUnmanaged& dp, const ExecViewUnmanaged& v); @@ -115,6 +151,7 @@ struct ComposeTransportImpl { void advance_hypervis_scalar(const Real dt); int run_trajectory_unit_tests(); + int run_enhanced_trajectory_unit_tests(); ComposeTransport::TestDepView::HostMirror test_trajectory(Real t0, Real t1, const bool independent_time_steps); @@ -122,8 +159,8 @@ struct ComposeTransportImpl { // avoid non-bfb-ness in, e.g., trig functions. void test_2d(const bool bfb, const int nstep, std::vector& eval); - template KOKKOS_INLINE_FUNCTION - static void loop_ijk (const KernelVariables& kv, const Fn& h) { + template KOKKOS_INLINE_FUNCTION + static void loop_ijk (const int KLIM, const KernelVariables& kv, const Fn& h) { using Kokkos::parallel_for; if (OnGpu::value) { const auto ttr = Kokkos::TeamThreadRange(kv.team, NP*NP); @@ -150,6 +187,11 @@ struct ComposeTransportImpl { } } + template KOKKOS_INLINE_FUNCTION + static void loop_ijk (const KernelVariables& kv, const Fn& h) { + loop_ijk(KLIM, kv, h); + } + template KOKKOS_INLINE_FUNCTION static void loop_ij (const KernelVariables& kv, const Fn& h) { if (OnGpu::value) { @@ -250,10 +292,113 @@ struct ComposeTransportImpl { return h; } + static KOKKOS_INLINE_FUNCTION + Real* pack2real (Scalar* pack) { return &(*pack)[0]; } + static KOKKOS_INLINE_FUNCTION + const Real* pack2real (const Scalar* pack) { return &(*pack)[0]; } template static KOKKOS_INLINE_FUNCTION - Real* pack2real (const View& v) { return &(*v.data())[0]; } + Real* pack2real (const View& v) { return pack2real(v.data()); } template static KOKKOS_INLINE_FUNCTION - const Real* cpack2real (const View& v) { return &(*v.data())[0]; } + const Real* cpack2real (const View& v) { return pack2real(v.data()); } + + KOKKOS_FUNCTION + static void ugradv_sphere ( + const SphereOperators& sphere_ops, const KernelVariables& kv, + const typename ViewConst >::type& vec_sphere2cart, + // velocity, latlon + const typename ViewConst >::type& u, + const typename ViewConst >::type& v, + const ExecViewUnmanaged& v_cart, + const ExecViewUnmanaged& ugradv_cart, + // [u dot grad] v, latlon + const ExecViewUnmanaged& ugradv) + { + for (int d_cart = 0; d_cart < 3; ++d_cart) { + const auto f1 = [&] (const int i, const int j, const int k) { + v_cart(i,j,k) = (vec_sphere2cart(0,d_cart,i,j) * v(0,i,j,k) + + vec_sphere2cart(1,d_cart,i,j) * v(1,i,j,k)); + }; + loop_ijk(kv, f1); + kv.team_barrier(); + + sphere_ops.gradient_sphere(kv, v_cart, ugradv_cart); + + const auto f2 = [&] (const int i, const int j, const int k) { + if (d_cart == 0) ugradv(0,i,j,k) = ugradv(1,i,j,k) = 0; + for (int d_latlon = 0; d_latlon < 2; ++d_latlon) + ugradv(d_latlon,i,j,k) += + vec_sphere2cart(d_latlon,d_cart,i,j)* + (u(0,i,j,k) * ugradv_cart(0,i,j,k) + u(1,i,j,k) * ugradv_cart(1,i,j,k)); + }; + loop_ijk(kv, f2); + } + } + + // Form a 3rd-degree Lagrange polynomial over (x(k-1:k+1), y(k-1:k+1)) and set + // yi(k) to its derivative at x(k). yps(:,:,0) is not written. + template + KOKKOS_FUNCTION static Real approx_derivative ( + const Real& xkm1, const Real& xk, const Real& xkp1, + const Real& ykm1, const Real& yk, const Real& ykp1) + { + return (ykm1*(( 1 /(xkm1 - xk ))*((xk - xkp1)/(xkm1 - xkp1))) + + yk *(( 1 /(xk - xkm1))*((xk - xkp1)/(xk - xkp1)) + + ((xk - xkm1)/(xk - xkm1))*( 1 /(xk - xkp1))) + + ykp1*(((xk - xkm1)/(xkp1 - xkm1))*( 1 /(xkp1 - xk )))); + } + + KOKKOS_INLINE_FUNCTION static void approx_derivative ( + const KernelVariables& kv, const CSNlevp& xs, const CSNlevp& ys, + const SNlev& yps) // yps(:,:,0) is undefined + { + CRNlevp x(cpack2real(xs)); + CRNlevp y(cpack2real(ys)); + RNlev yp(pack2real(yps)); + const auto f = [&] (const int i, const int j, const int k) { + if (k == 0) return; + const auto& xkm1 = x(i,j,k-1); + const auto& xk = x(i,j,k ); // also the interpolation point + const auto& xkp1 = x(i,j,k+1); + yp(i,j,k) = approx_derivative(x(i,j,k-1), x(i,j,k), x(i,j,k+1), + y(i,j,k-1), y(i,j,k), y(i,j,k+1)); + }; + loop_ijk(kv, f); + } + + template + KOKKOS_FUNCTION static void calc_eta_dot_dpdn ( + const KernelVariables& kv, + const HyBiPackT& hybrid_bi, // const Scalar[NUM_LEV_P] + // divergence_sphere of (v dp) at midpoints, scalar + const DivDpScalT& divdps, + // eta_dot_dpdn at interfaces, pack and scalar views of same data + const EddPackT& edd, const EddScalT& edds) + { + const auto ttr = Kokkos::TeamThreadRange(kv.team, NP*NP); + const auto tvr = Kokkos::ThreadVectorRange(kv.team, NUM_LEV); + const auto f = [&] (const int idx) { + const int i = idx / NP, j = idx % NP; + const auto r = [&] (const int k, Real& dps, const bool final) { + assert(k != 0 || dps == 0); + if (final) edds(i,j,k) = dps; + dps += divdps(i,j,k); + }; + Dispatch<>::parallel_scan(kv.team, num_phys_lev, r); + const int kend = num_phys_lev - 1; + const Real dps = edds(i,j,kend) + divdps(i,j,kend); + assert(hybrid_bi(0)[0] == 0); + const auto s = [&] (const int kp) { + edd(i,j,kp) = hybrid_bi(kp)*dps - edd(i,j,kp); + if (kp == 0) edd(i,j,kp)[0] = 0; + }; + Kokkos::parallel_for(tvr, s); + assert(edds(i,j,0) == 0); + const int bottom = num_phys_lev; + edds(i,j,bottom) = 0; // benign write race + }; + Kokkos::parallel_for(ttr, f); + } }; } // namespace Homme diff --git a/components/homme/src/share/cxx/ComposeTransportImplEnhancedTrajectory.cpp b/components/homme/src/share/cxx/ComposeTransportImplEnhancedTrajectory.cpp new file mode 100644 index 000000000000..62bd190bc305 --- /dev/null +++ b/components/homme/src/share/cxx/ComposeTransportImplEnhancedTrajectory.cpp @@ -0,0 +1,523 @@ +/******************************************************************************** + * HOMMEXX 1.0: Copyright of Sandia Corporation + * This software is released under the BSD license + * See the file 'COPYRIGHT' in the HOMMEXX/src/share/cxx directory + *******************************************************************************/ + +#include "Config.hpp" +#ifdef HOMME_ENABLE_COMPOSE + +/* This is the second trajectory method for semi-Lagrangian transport; the first + is in ComposeTransportImplTrajectory.cpp. + + Usage + + To use it, add the following setting to the Homme namelist: + semi_lagrange_trajectory_nsubstep = N ! where N > 0 + A value of 0 (default) means Homme will use the original method. + + Another option makes the method use more than two velocity snapshots per time + step: + semi_lagrange_trajectory_nvelocity = N ! where N > 2 + A value <= 2 other than -1 means Homme will use the standard two velocity + snapshots (time step end points). -1 (default) triggers an internal + calculation based on nsubstep. This option has no effect if nsubstep=0 + (default). + + Summary + + This method provides multiple benefits over the original method, depending on + configuration: + * Supports (much) longer time steps than the original method. + * Maximizes flexibility in specifying the various atmosphere time steps. + * Greater accuracy than the original method for time steps the original + method can handle. + * Extreme accuracy in hypothetical niche applications. + + Method overview + + Recall that semi-Lagrangian tracer transport has six phases: + 1. At time step n+1, for each GLL point on the Eulerian grid, compute a + trajectory backward in time to the departure point at time n. This step is + independent of number of tracers. + 2. Simultaneously reconstruct vertically Lagrangian levels at time + n+1. This step is independent of number of tracers. + 3. In each level, for each level-midpoint departure point, interpolate tracer + mixing ratios at time n to the point. These mixing ratios are then the new + ratios at time n+1 on the Eulerian grid. + 4. Optionally apply hyperviscosity. + 5. Apply the Communication-Efficient Density Reconstructor (CEDR). + 6. Vertically remap tracers at time n+1 from the reconstructed vertically + Lagrangian levels to the vertically Eulerian grid. + + Trajectory methods implement phases 1 and 2. + + The key capability of this enhanced trajectory method (ETM) is to be able to + take multiple substeps when computing the departure points. The original + method cannot substep. Each substep has second-order accuracy, so the overall + method is always second-order accurate. But as the number of substeps + increases, so does accuracy. + + A second capability of the ETM is to use more velocity snapshots than just + the tracer time step end-point snapshots. For example, if there are two + substeps, the method can use three velocity snapshots: beginning, middle, + end. The first substep uses (middle, end), and the second uses (beginning, + middle). (This might be the opposite of what you expected, but recall that + the trajectory is computed backward in time.) + + At a software level, a third capability is to use an arbitrarily large + element halo when computing departure points. The original method is limited + to two halo layers. Extra halo layers do not increase the cost of search for + a fixed time step because of the ordering of elements in the layer. + + Speedup comes from the fact that taking a longer time step means phases 3-6, + the most expensive phases, run less often. Phases 1 and 2 also run less often + but take more time per run, summing to about the same cost over a fixed time + duration T as the original method. + + Algorithm outline + + This method works principally in the eta coordinate. eta is constant in a + level on the Eulerian grid. + Let time t1 > t0 and consider a trajectory substep from t1 to t0. + Terminology: An arrival point is the time-t1 point of a trajectory. A + departure point is the time-t0 point. + For each interface node at times t0 and t1, compute eta_dot dp/deta + (calc_eta_dot_dpdn). + For each midpoint node at times t0 and t1, compute + eta_dot = eta_dot dp/deta/(A_eta p0 + B_eta ps) + (calc_etadotmid_from_etadotdpdnint). + Use eta_dot, the horizontal velocity data, and the update formula described + in the comment for calc_nodal_velocities to compute the velocity term in the + update formula at the Eulerian vertical-midpoint nodes. Call the result V + (calc_vel_horiz_formula_node_ref_mid, calc_eta_dot_formula_node_ref_mid). + In general, the trajectory arrival point at t1 is not on the grid, but it + is in the first substep. If it is not on the grid, interpolate V to the + arrival points to produce V_dep (calc_v_departure). A detail here is we + should actually think of the original velocity data as being interpolated, + and then V_dep is computed from the interpolated data. But the formula to + compute V is linear in these data, so we can defer interpolation to the end + and do it just once. + Integrate the trajectory backward from t1 at the arrival point to t0 at the + departure point using V_dep (update_dep_points). + The algorithm up to this point can be substepped, running multiple times to + go backward from t1 to t0 in multiple steps. + After substepping is finished, there is one final part. + So far we have computed departure points corresponding to Eulerian-grid + arrival points. But now we need to account for the fact that the levels are + vertically Lagrangian ("floating"). The arrival points we actually need are + those on the floating levels at time t1 corresponding to Eulerian levels at + time t0. This is implemented in + describe interp_departure_points_to_floating_level_midpoints. + We use the following notation: yi = I[y(x)](xi) is an interpolant + constructed from y(x) and evaluated at xi. + On input, we have departure-point level-midpoint eta, eta_dep_mid, and the + corresponding horizontal position, p_dep_mid. We also know eta level + interfaces and midpoints on the reference grid, eta_ref_int and eta_ref_mid, + and the top and bottom boundary values of eta, eta(0) and eta(1). + First, reconstruct Lagrangian levels at t1 on the arrival column: + eta_arr_int = I[eta_ref_mid([eta(0),eta_dep_mid,eta(1)])](eta_ref_int). + Second, compute the Lagrangian level midpoints at t1 on the arrival column: + eta_arr_mid = I[eta_ref_mid([eta(0),eta_dep_mid,eta(1)])](eta_ref_mid). + Third, compute the departure horizontal points corresponding to the arrival + Lagrangian level midpoints: + p_dep_mid(eta_arr_mid) = I[p_dep_mid(eta_ref_mid)](eta_arr_mid). + These calcualtions provide us with the final results: p_dep_mid is used in + phase 3, and eta_arr_int is used in phase 6. + */ + +#include "ComposeTransportImplEnhancedTrajectoryImpl.hpp" + +namespace Homme { +namespace { + +// Set dep_points_all to level-midpoint arrival points. +void init_dep_points (const CTI& c, const cti::DeparturePoints& dep_pts) { + const auto independent_time_steps = c.m_data.independent_time_steps; + const auto& sphere_cart = c.m_geometry.m_sphere_cart; + const CRNV hyetam(cti::cpack2real(c.m_hvcoord.etam)); + assert(not independent_time_steps or dep_pts.extent_int(4) == 4); + const auto f = KOKKOS_LAMBDA (const int idx) { + int ie, lev, i, j; + cti::idx_ie_physlev_ij(idx, ie, lev, i, j); + for (int d = 0; d < 3; ++d) + dep_pts(ie,lev,i,j,d) = sphere_cart(ie,i,j,d); + if (independent_time_steps) + dep_pts(ie,lev,i,j,3) = hyetam(lev); + }; + c.launch_ie_physlev_ij(f); +} + +void update_dep_points ( + const CTI& c, const Real dtsub, const cti::DeparturePoints& vdep, + const cti::DeparturePoints& dep_pts) +{ + const auto independent_time_steps = c.m_data.independent_time_steps; + const auto is_sphere = c.m_data.geometry_type == 0; + const auto scale_factor = c.m_geometry.m_scale_factor; + const auto f = KOKKOS_LAMBDA (const int idx) { + int ie, lev, i, j; + cti::idx_ie_physlev_ij(idx, ie, lev, i, j); + // Update horizontal position. + Real p[3]; + for (int d = 0; d < 3; ++d) + p[d] = dep_pts(ie,lev,i,j,d) - dtsub*vdep(ie,lev,i,j,d)/scale_factor; + if (is_sphere) { + const auto norm = std::sqrt(square(p[0]) + square(p[1]) + square(p[2])); + for (int d = 0; d < 3; ++d) + p[d] /= norm; + } + for (int d = 0; d < 3; ++d) + dep_pts(ie,lev,i,j,d) = p[d]; + if (independent_time_steps) { + // Update vertical position. + dep_pts(ie,lev,i,j,3) -= dtsub*vdep(ie,lev,i,j,3); + } + }; + c.launch_ie_physlev_ij(f); +} + +/* Evaluate a formula to provide an estimate of nodal velocities that are use to + create a 2nd-order update to the trajectory. The fundamental formula for the + update in position p from arrival point p1 to departure point p0 is + p0 = p1 - dt/2 (v(p1,t0) + v(p1,t1) - dt v(p1,t1) grad v(p1,t0)). + Here we compute the velocity estimate at the nodes: + 1/2 (v(p1,t0) + v(p1,t1) - dt v(p1,t1) grad v(p1,t0)). +*/ +void calc_nodal_velocities ( + const CTI& c, const Real dtsub, const Real halpha[2], + const cti::CVSlot& v1, const cti::CDpSlot& dp1, const int idx1, + const cti::CVSlot& v2, const cti::CDpSlot& dp2, const int idx2, + const cti::DeparturePoints& vnode) +{ + using Kokkos::ALL; + const auto& d = c.m_data; + const auto& h = c.m_hvcoord; + const auto& sphere_ops = c.m_sphere_ops; + const auto& vec_sph2cart = c.m_geometry.m_vec_sph2cart; + const bool independent_time_steps = d.independent_time_steps; + const auto ps0 = h.ps0; + const auto hyai0 = h.hybrid_ai0; + const auto& hybi = h.hybrid_bi_packed; + const auto& hydai = h.hybrid_ai_delta; + const auto& hydbi = h.hybrid_bi_delta; + const auto& hyetam = h.etam; + const auto& hyetai = h.etai; + const auto& hydetai = d.hydetai; + const auto& buf1a = d.buf1o[0]; const auto& buf1b = d.buf1o[1]; + const auto& buf1c = d.buf1o[2]; const auto& buf1d = d.buf1o[3]; + const auto& buf2a = d.buf2 [0]; const auto& buf2b = d.buf2 [1]; + const auto& buf2c = d.buf2 [2]; const auto& buf2d = d.buf2 [3]; + const auto alpha0 = halpha[0], alpha1 = halpha[1]; + const auto f = KOKKOS_LAMBDA (const cti::MT& team) { + KernelVariables kv(team); + const int ie = kv.ie; + const auto wrk1 = Homme::subview(buf1a, kv.team_idx); + const auto wrk2 = Homme::subview(buf1b, kv.team_idx); + const auto vwrk1 = Homme::subview(buf2a, kv.team_idx); + const auto vwrk2 = Homme::subview(buf2b, kv.team_idx); + const auto v1_ie = Homme::subview(v1, ie, idx1); + const auto v2_ie = Homme::subview(v2, ie, idx2); + const Real alpha[] = {alpha0, alpha1}; + CSelNlevp eta_dot[] = {Homme::subview(buf1c, kv.team_idx), + Homme::subview(buf1d, kv.team_idx)}; + { + SelNlevp eta_dot[] = {Homme::subview(buf1c, kv.team_idx), + Homme::subview(buf1d, kv.team_idx)}; + if (independent_time_steps) { + const auto dp1_ie = Homme::subview(dp1, ie, idx1); + const auto dp2_ie = Homme::subview(dp2, ie, idx2); + calc_eta_dot_ref_mid(kv, sphere_ops, + ps0, hyai0, hybi, hydai, hydbi, hydetai, + alpha, v1_ie, dp1_ie, v2_ie, dp2_ie, + wrk1, wrk2, vwrk1, + eta_dot); + } else { + for (int t = 0; t < 2; ++t) { + const auto& ed = eta_dot[t]; + const auto f = [&] (const int i, const int j, const int k) { + ed(i,j,k) = 0; + }; + cti::loop_ijk(kv, f); + } + } + } + // Collect the horizontal nodal velocities. v1,2 are on Eulerian levels. v1 + // is from time t1 < t2. + auto* vm1 = Homme::subview(buf2c, kv.team_idx).data(); + auto* vm2 = Homme::subview(buf2d, kv.team_idx).data(); + CS2elNlev vsph[] = {CS2elNlev(vm1), CS2elNlev(vm2)}; + { + S2elNlev vsph[] = {S2elNlev(vm1), S2elNlev(vm2)}; + for (int t = 0; t < 2; ++t) { + const auto& v = vsph[t]; + for (int d = 0; d < 2; ++d) { + const auto f = [&] (const int i, const int j, const int k) { + v(d,i,j,k) = ((1 - alpha[t])*v1_ie(d,i,j,k) + + /**/ alpha[t] *v2_ie(d,i,j,k)); + }; + cti::loop_ijk(kv, f); + } + } + } + kv.team_barrier(); + // Given the vertical and horizontal nodal velocities at time endpoints, + // evaluate the velocity estimate formula, providing the final horizontal + // and vertical velocity estimates at midpoint nodes. + const auto vnode_ie = Kokkos::subview(vnode, ie, ALL,ALL,ALL,ALL); + const auto vec_sph2cart_ie = Homme::subview(vec_sph2cart, ie); + calc_vel_horiz_formula_node_ref_mid(kv, sphere_ops, + hyetam, vec_sph2cart_ie, + dtsub, vsph, eta_dot, + wrk1, vwrk1, vwrk2, + vnode_ie); + if (independent_time_steps) { + kv.team_barrier(); + calc_eta_dot_formula_node_ref_mid(kv, sphere_ops, + hyetai, hyetam, + dtsub, vsph, eta_dot, + wrk1, vwrk1, + vnode_ie); + } + }; + Kokkos::parallel_for(c.m_tp_ne, f); +} + +// Determine the departure points corresponding to the vertically Lagrangian +// grid's arrival midpoints, where the floating levels are those that evolve +// over the course of the full tracer time step. Also compute divdp, which holds +// the floating levels' dp values for later use in vertical remap. +void interp_departure_points_to_floating_level_midpoints (const CTI& c, const int np1) { + using Kokkos::ALL; + const int nlev = NUM_PHYSICAL_LEV, nlevp = nlev+1; + const auto is_sphere = c.m_data.geometry_type == 0; + const auto& d = c.m_data; + const auto& h = c.m_hvcoord; + const auto ps0 = h.ps0; + const auto hyai0 = h.hybrid_ai0; + const auto& hybi = h.hybrid_bi; + const auto& hyetai = h.etai; + const CRNV hyetam(cti::cpack2real(h.etam)); + const auto& detam_ref = d.hydetam_ref; + const auto deta_tol = d.deta_tol; + const auto& dep_pts = d.dep_pts; + const auto& dp3d = c.m_state.m_dp3d; + const auto& buf1a = d.buf1e[0]; const auto& buf1b = d.buf1e[1]; + const auto& buf1c = d.buf1e[2]; const auto& buf1d = d.buf1e[3]; + const auto& buf2a = d.buf2[0]; + const auto f = KOKKOS_LAMBDA (const cti::MT& team) { + KernelVariables kv(team); + const int ie = kv.ie; + const auto wrk1 = Homme::subview(buf1a, kv.team_idx); + const auto wrk2 = Homme::subview(buf1b, kv.team_idx); + const auto wrk3 = Homme::subview(buf1c, kv.team_idx); + const auto wrk4 = Homme::subview(buf1d, kv.team_idx); + const auto vwrk = Homme::subview(buf2a, kv.team_idx); + // Reconstruct Lagrangian levels at t1 on arrival column: + // eta_arr_int = I[eta_ref_mid([eta(0),eta_dep_mid,eta(1)])](eta_ref_int) + const auto etam = p2rel(wrk3.data(), nlev); + const auto f = [&] (const int i, const int j, const int k) { + etam(i,j,k) = dep_pts(ie,k,i,j,3); + }; + cti::loop_ijk(kv, f); + kv.team_barrier(); + limit_etam(kv, nlev, + hyetai, detam_ref, deta_tol, + p2rel(wrk1.data(), nlevp), p2rel(wrk2.data(), nlevp), + etam); + kv.team_barrier(); + { + // Compute eta_arr_int. + const auto etai_arr = p2rel(wrk4.data(), nlevp); + eta_interp_eta(kv, nlev, + hyetai, + etam, hyetam, + p2rel(wrk1.data(), nlev+2), RnV(cti::pack2real(wrk2), nlev+2), + nlevp-2, hyetai, etai_arr, 1); + const auto f = [&] (const int i, const int j) { + etai_arr(i,j,0) = hyetai(0); + etai_arr(i,j,nlev) = hyetai(nlev); + }; + c.loop_ij(kv, f); + // Compute divdp. + const ExecViewUnmanaged ps(cti::pack2real(vwrk)); + calc_ps(kv, nlev, + ps0, hyai0, + Homme::subview(dp3d, ie, np1), + ps); + kv.team_barrier(); + eta_to_dp(kv, nlev, + ps0, hybi, hyetai, + ps, etai_arr, + p2rel(wrk2.data(), nlev+1), + RelnV(cti::pack2real(Homme::subview(c.m_derived.m_divdp, ie)), + NP, NP, NUM_LEV*VECTOR_SIZE)); + kv.team_barrier(); + } + // Compute Lagrangian level midpoints at t1 on arrival column: + // eta_arr_mid = I[eta_ref_mid([eta(0),eta_dep_mid,eta(1)])](eta_ref_mid) + const auto etam_arr = p2rel(wrk4.data(), nlev); + eta_interp_eta(kv, nlev, + hyetai, + etam, hyetam, + p2rel(wrk1.data(), nlev+2), RnV(cti::pack2real(wrk2), nlev+2), + nlev, hyetam, etam_arr); + kv.team_barrier(); + // Compute departure horizontal points corresponding to arrival + // Lagrangian level midpoints: + // p_dep_mid(eta_arr_mid) = I[p_dep_mid(eta_ref_mid)](eta_arr_mid) + { + const RelnV dpts_in(cti::pack2real(vwrk), NP, NP, nlev); + const RelnV dpts_out(dpts_in.data() + NP*NP*nlev, NP, NP, nlev); + for (int d = 0; d < 3; ++d) { + const auto f = [&] (const int i, const int j, const int k) { + dpts_in(i,j,k) = dep_pts(ie,k,i,j,d); + }; + c.loop_ijk(kv, f); + kv.team_barrier(); + eta_interp_horiz(kv, nlev, + hyetai, + hyetam, dpts_in, + RnV(cti::pack2real(wrk2), nlev+2), p2rel(wrk1.data(), nlev+2), + etam_arr, dpts_out); + kv.team_barrier(); + const auto g = [&] (const int i, const int j, const int k) { + dep_pts(ie,k,i,j,d) = dpts_out(i,j,k); + }; + c.loop_ijk(kv, g); + kv.team_barrier(); + } + if (is_sphere) { + // Normalize. + const auto h = [&] (const int i, const int j, const int k) { + Real norm = 0; + for (int d = 0; d < 3; ++d) norm += square(dep_pts(ie,k,i,j,d)); + norm = std::sqrt(norm); + for (int d = 0; d < 3; ++d) dep_pts(ie,k,i,j,d) /= norm; + }; + c.loop_ijk(kv, h); + } + } + }; + Kokkos::parallel_for(c.m_tp_ne, f); +} + +void dss_vnode (const CTI& c, const cti::DeparturePoints& vnode) { + const int ndim = c.m_data.independent_time_steps ? 4 : 3; + const auto& spheremp = c.m_geometry.m_spheremp; + const auto& rspheremp = c.m_geometry.m_rspheremp; + const auto& vp = c.m_tracers.qtens_biharmonic; + const ExecViewUnmanaged + v(cti::pack2real(vp), vp.extent_int(0), vp.extent_int(1)); + const auto f = KOKKOS_LAMBDA (const int idx) { + int ie, lev, i, j; + cti::idx_ie_physlev_ij(idx, ie, lev, i, j); + for (int d = 0; d < ndim; ++d) + v(ie,d,i,j,lev) = vnode(ie,lev,i,j,d)*spheremp(ie,i,j)*rspheremp(ie,i,j); + }; + c.launch_ie_physlev_ij(f); + Kokkos::fence(); + const auto be = c.m_v_dss_be[c.m_data.independent_time_steps ? 1 : 0]; + be->exchange(); + Kokkos::fence(); + const auto g = KOKKOS_LAMBDA (const int idx) { + int ie, lev, i, j; + cti::idx_ie_physlev_ij(idx, ie, lev, i, j); + for (int d = 0; d < ndim; ++d) + vnode(ie,lev,i,j,d) = v(ie,d,i,j,lev); + }; + c.launch_ie_physlev_ij(g); +} + +} // namespace anon + +// For limit_etam. +void ComposeTransportImpl::setup_enhanced_trajectory () { + const auto etai = cmvdc(m_hvcoord.etai); + const Real deta_ave = (etai(num_phys_lev) - etai(0)) / num_phys_lev; + m_data.deta_tol = 10*std::numeric_limits::epsilon()*deta_ave; + + // diff(etai) + m_data.hydetai = decltype(m_data.hydetai)("hydetai"); + const auto detai_pack = Kokkos::create_mirror_view(m_data.hydetai); + HostViewUnmanaged detai(pack2real(detai_pack)); + for (int k = 0; k < num_phys_lev; ++k) + detai(k) = etai(k+1) - etai(k); + Kokkos::deep_copy(m_data.hydetai, detai_pack); + + const auto etamp = cmvdc(m_hvcoord.etam); + HostViewUnmanaged etam(pack2real(etamp)); + + // hydetam_ref. + m_data.hydetam_ref = decltype(m_data.hydetam_ref)("hydetam_ref"); + const auto m = Kokkos::create_mirror_view(m_data.hydetam_ref); + const int nlev = num_phys_lev; + m(0) = etam(0) - etai(0); + for (int k = 1; k < nlev; ++k) m(k) = etam(k) - etam(k-1); + m(nlev) = etai(nlev) - etam(nlev-1); + Kokkos::deep_copy(m_data.hydetam_ref, m); + + // etam + homme::compose::set_hvcoord(etai(0), etai(num_phys_lev), etam.data()); +} + +void ComposeTransportImpl::calc_enhanced_trajectory (const int np1, const Real dt) { + GPTLstart("compose_calc_enhanced_trajectory"); + + const auto& dep_pts = m_data.dep_pts; + const auto& vnode = m_data.vnode; + const auto& vdep = m_data.vdep; + + init_dep_points(*this, dep_pts); + + const int nelemd = m_data.nelemd; + const Real dtsub = dt / m_data.trajectory_nsubstep; + const int nsubstep = m_data.trajectory_nsubstep; + for (int step = 0; step < nsubstep; ++step) { + { + Kokkos::fence(); + GPTLstart("compose_vnode"); + const Real alpha[] = {Real(nsubstep-step-1)/nsubstep, + Real(nsubstep-step )/nsubstep}; + const CVSlot v1(m_derived.m_vstar.data(), nelemd, 1); + const CDpSlot dp1(m_derived.m_dp.data(), nelemd, 1); + const auto& v2 = m_state.m_v; + const auto& dp2 = m_state.m_dp3d; + calc_nodal_velocities(*this, dtsub, alpha, + v1, dp1, 0, v2, dp2, np1, + vnode); + Kokkos::fence(); + GPTLstop("compose_vnode"); + } + + GPTLstart("compose_v_bexchv"); + dss_vnode(*this, vnode); + Kokkos::fence(); + GPTLstop("compose_v_bexchv"); + + if (step == 0) { + update_dep_points(*this, dtsub, vnode, dep_pts); + } else { + GPTLstart("compose_vdep"); + homme::compose::calc_v_departure(step, dtsub); + Kokkos::fence(); + GPTLstop("compose_vdep"); + + update_dep_points(*this, dtsub, vdep, dep_pts); + } + } + Kokkos::fence(); + + if (m_data.independent_time_steps) { + GPTLstart("compose_floating_dep_pts"); + interp_departure_points_to_floating_level_midpoints(*this, np1); + Kokkos::fence(); + GPTLstop("compose_floating_dep_pts"); + } + + GPTLstop("compose_calc_enhanced_trajectory"); +} + +} // namespace Homme + +#endif // HOMME_ENABLE_COMPOSE diff --git a/components/homme/src/share/cxx/ComposeTransportImplEnhancedTrajectoryImpl.hpp b/components/homme/src/share/cxx/ComposeTransportImplEnhancedTrajectoryImpl.hpp new file mode 100644 index 000000000000..1d9ee6da4adf --- /dev/null +++ b/components/homme/src/share/cxx/ComposeTransportImplEnhancedTrajectoryImpl.hpp @@ -0,0 +1,720 @@ +/******************************************************************************** + * HOMMEXX 1.0: Copyright of Sandia Corporation + * This software is released under the BSD license + * See the file 'COPYRIGHT' in the HOMMEXX/src/share/cxx directory + *******************************************************************************/ + +#include "Config.hpp" +#ifdef HOMME_ENABLE_COMPOSE + +#ifndef HOMMEXX_COMPOSE_TRANSPORT_IMPL_ENHANCED_TRAJECTORY_IMPL_HPP +#define HOMMEXX_COMPOSE_TRANSPORT_IMPL_ENHANCED_TRAJECTORY_IMPL_HPP + +#include "ComposeTransportImpl.hpp" + +#include "compose_hommexx.hpp" + +namespace Homme { +namespace { // anon + +using cti = ComposeTransportImpl; +using CTI = ComposeTransportImpl; +using CSelNlev = cti::CSNlev; +using CRelNlev = cti::CRNlev; +using CSelNlevp = cti::CSNlevp; +using CRelNlevp = cti::CRNlevp; +using CS2elNlev = cti::CS2Nlev; +using CR2elNlev = cti::CR2Nlev; +using SelNlev = cti::SNlev; +using RelNlev = cti::RNlev; +using SelNlevp = cti::SNlevp; +using RelNlevp = cti::RNlevp; +using S2elNlev = cti::S2Nlev; +using R2elNlev = cti::R2Nlev; +using S2elNlevp = cti::S2Nlevp; + +using RelV = ExecViewUnmanaged; +using CRelV = typename ViewConst::type; + +template using SelNV = ExecViewUnmanaged; +template using CSelNV = typename ViewConst>::type; + +template using RelNV = ExecViewUnmanaged; +template using CRelNV = typename ViewConst>::type; + +template using RNV = ExecViewUnmanaged; +template using CRNV = typename ViewConst>::type; +using RNlevp = RNV; +using CRNlevp = CRNV; + +using RnV = ExecViewUnmanaged; +using CRnV = ExecViewUnmanaged; +using SnV = ExecViewUnmanaged; +using CSnV = ExecViewUnmanaged; + +template using SNV = ExecViewUnmanaged; +template using CSNV = typename ViewConst>::type; + +using RelnV = ExecViewUnmanaged; +using CRelnV = ExecViewUnmanaged; +using SelnV = ExecViewUnmanaged; +using CSelnV = ExecViewUnmanaged; + +// Helper functions to move between various array data structures and assert +// things about them. + +KOKKOS_INLINE_FUNCTION +static int calc_npack (const int nscal) { + return (nscal + cti::packn - 1) / VECTOR_SIZE; +} + +KOKKOS_INLINE_FUNCTION +static int calc_nscal (const int npack) { + return npack * VECTOR_SIZE; +} + +KOKKOS_INLINE_FUNCTION +RnV getcol (const RelnV& a, const int i, const int j) { + return Kokkos::subview(a,i,j,Kokkos::ALL); +} + +KOKKOS_INLINE_FUNCTION +CRnV getcolc (const CRelnV& a, const int i, const int j) { + return Kokkos::subview(a,i,j,Kokkos::ALL); +} + +KOKKOS_INLINE_FUNCTION +RelnV elp2r (const SelnV& p) { + return RelnV(cti::pack2real(p), NP, NP, calc_nscal(p.extent_int(2))); +} + +KOKKOS_INLINE_FUNCTION +CRelnV elp2r (const CSelnV& p) { + return CRelnV(cti::cpack2real(p), NP, NP, calc_nscal(p.extent_int(2))); +} + +KOKKOS_INLINE_FUNCTION +RelnV p2rel (Scalar* data, const int nlev) { + return RelnV(cti::pack2real(data), NP, NP, nlev); +} + +KOKKOS_INLINE_FUNCTION +void assert_eln (const CRelnV& a, const int nlev) { + assert(a.extent_int(0) >= NP); + assert(a.extent_int(1) >= NP); + assert(a.extent_int(2) >= nlev); +} + +KOKKOS_INLINE_FUNCTION +void assert_eln (const CSelnV& a, const int nlev) { + assert(a.extent_int(0) >= NP); + assert(a.extent_int(1) >= NP); + assert(calc_nscal(a.extent_int(2)) >= nlev); +} + +// Find the support for the linear interpolant. +// For sorted ascending x[0:n] and x in [x[0], x[n-1]] with hint xi_idx, +// return i such that x[i] <= xi <= x[i+1]. +// This function is meant for the case that x_idx is very close to the +// support. If that isn't true, then this method is inefficient; binary search +// should be used instead. +template +KOKKOS_FUNCTION static +int find_support (const int n, const ConstRealArray& x, const int x_idx, + const Real xi) { + assert(xi >= x[0] and xi <= x[n-1]); + // Handle the most common case. + if (x_idx < n-1 and xi >= x[x_idx ] and xi <= x[x_idx+1]) return x_idx; + if (x_idx > 0 and xi >= x[x_idx-1] and xi <= x[x_idx ]) return x_idx-1; + // Move on to less common ones. + const int max_step = max(x_idx, n-1 - x_idx); + for (int step = 1; step <= max_step; ++step) { + if (x_idx < n-1-step and xi >= x[x_idx+step ] and xi <= x[x_idx+step+1]) + return x_idx+step; + if (x_idx > step and xi >= x[x_idx-step-1] and xi <= x[x_idx-step ]) + return x_idx-step-1; + } + assert(false); + return -1; +} + +// Linear interpolation core formula. +template +KOKKOS_FUNCTION Real +linterp (const int n, const XT& x, const YT& y, const int x_idx, const Real xi) { + const auto isup = find_support(n, x, x_idx, xi); + const Real a = (xi - x[isup])/(x[isup+1] - x[isup]); + return (1-a)*y[isup] + a*y[isup+1]; +} + +// Linear interpolation at the lowest level of team ||ism. +// Range provides this ||ism over index 0 <= k < ni. +// Interpolate y(x) to yi(xi). +// x_idx_offset is added to k in the call to find_support. +// Arrays should all have rank 1. +// Notation: yi = I[y(x)](xi) is an interpolant constructed from y(x) and +// evaluated at xi. +template +KOKKOS_FUNCTION void +linterp (const Range& range, + const int n , const XT& x , const YT& y, + const int ni, const XIT& xi, const YIT& yi, + const int x_idx_offset = 0, const char* const caller = nullptr) { +#ifndef NDEBUG + if (xi[0] < x[0] or xi[ni-1] > x[n-1]) { + if (caller) + printf("linterp: xi out of bounds: %s %1.15e %1.15e %1.15e %1.15e\n", + caller ? caller : "NONE", x[0], xi[0], xi[ni-1], x[n-1]); + assert(false); + } +#endif + assert(range.start == 0); + assert(range.end == ni); + const auto f = [&] (const int k) { + yi[k] = linterp(n, x, y, k + x_idx_offset, xi[k]); + }; + Kokkos::parallel_for(range, f); +} + +// Compute Lagrangian level midpoints at t1 on arrival column: +// eta_arr_mid = I[eta_ref_mid([eta(0),eta_dep_mid,eta(1)])](eta_ref_mid). +KOKKOS_FUNCTION void +eta_interp_eta (const KernelVariables& kv, const int nlev, + const CRnV& hy_etai, const CRelnV& x, const CRnV& y, + const RelnV& xwrk, const RnV& ywrk, + // Use xi(i_os:), yi(i,j,i_os:). + const int ni, const CRnV& xi, const RelnV& yi, const int i_os = 0) { + const auto& xbdy = xwrk; + const auto& ybdy = ywrk; + assert(hy_etai.extent_int(0) >= nlev+1); + assert_eln(x, nlev); + assert(y.extent_int(0) >= nlev); + assert_eln(xbdy, nlev+2); + assert(ybdy.extent_int(0) >= nlev+2); + assert(xi.extent_int(0) >= i_os + ni); + assert_eln(yi, i_os + ni); + const auto ttr = Kokkos::TeamThreadRange(kv.team, NP*NP); + const auto tvr_ni = Kokkos::ThreadVectorRange(kv.team, ni); + const auto tvr_nlevp2 = Kokkos::ThreadVectorRange(kv.team, nlev+2); + const auto f_y = [&] (const int k) { + ybdy(k) = (k == 0 ? hy_etai(0) : + k == nlev+1 ? hy_etai(nlev) : + /**/ y(k-1)); + }; + Kokkos::parallel_for(Kokkos::TeamVectorRange(kv.team, nlev+2), f_y); + kv.team_barrier(); + const auto f_x = [&] (const int idx) { + const int i = idx / NP, j = idx % NP; + const auto g = [&] (const int k) { + xbdy(i,j,k) = (k == 0 ? hy_etai(0) : + k == nlev+1 ? hy_etai(nlev) : + /**/ x(i,j,k-1)); + }; + Kokkos::parallel_for(tvr_nlevp2, g); + }; + Kokkos::parallel_for(ttr, f_x); + kv.team_barrier(); + const auto f_linterp = [&] (const int idx) { + const int i = idx / NP, j = idx % NP; + linterp(tvr_ni, + nlev+2, getcolc(xbdy,i,j), ybdy, + ni, xi.data() + i_os, getcol(yi,i,j).data() + i_os, + 1, "eta_interp_eta"); + }; + Kokkos::parallel_for(ttr, f_linterp); +} + +// Compute departure horizontal points corresponding to arrival Lagrangian level +// midpoints: +// p_dep_mid(eta_arr_mid) = I[p_dep_mid(eta_ref_mid)](eta_arr_mid) +KOKKOS_FUNCTION void +eta_interp_horiz (const KernelVariables& kv, const int nlev, + const CRnV& hy_etai, const CRnV& x, const CRelnV& y, + const RnV& xwrk, const RelnV& ywrk, + const CRelnV& xi, const RelnV& yi) { + const auto& xbdy = xwrk; + const auto& ybdy = ywrk; + assert(hy_etai.extent_int(0) >= nlev+1); + assert(x.extent_int(0) >= nlev); + assert_eln(y, nlev); + assert(xbdy.extent_int(0) >= nlev+2); + assert_eln(ybdy, nlev+2); + assert_eln(xi, nlev); + assert_eln(yi, nlev); + const auto ttr = Kokkos::TeamThreadRange(kv.team, NP*NP); + const auto tvr_nlev = Kokkos::ThreadVectorRange(kv.team, nlev); + const auto tvr_nlevp2 = Kokkos::ThreadVectorRange(kv.team, nlev+2); + const auto f_x = [&] (const int k) { + xbdy(k) = (k == 0 ? hy_etai(0) : + k == nlev+1 ? hy_etai(nlev) : + /**/ x(k-1)); + }; + Kokkos::parallel_for(Kokkos::TeamVectorRange(kv.team, nlev+2), f_x); + kv.team_barrier(); + const auto f_y = [&] (const int idx) { + const int i = idx / NP, j = idx % NP; + const auto g = [&] (const int k) { + // Constant interp outside of the etam support. + ybdy(i,j,k) = (k == 0 ? y(i,j,0) : + k == nlev+1 ? y(i,j,nlev-1) : + /**/ y(i,j,k-1)); + }; + Kokkos::parallel_for(tvr_nlevp2, g); + }; + Kokkos::parallel_for(ttr, f_y); + kv.team_barrier(); + const auto f_linterp = [&] (const int idx) { + const int i = idx / NP, j = idx % NP; + linterp(tvr_nlev, + nlev+2, xbdy, getcolc(ybdy,i,j), + nlev, getcolc(xi,i,j), getcol(yi,i,j), + 1, "eta_interp_horiz"); + }; + Kokkos::parallel_for(ttr, f_linterp); +} + +/* Compute level pressure thickness given eta at interfaces using the following + approximation: + e = A(e) + B(e) + p(e) = A(e) p0 + B(e) ps + = e p0 + B(e) (ps - p0) + a= e p0 + I[Bi(eref)](e) (ps - p0). + Then dp = diff(p). +*/ +KOKKOS_FUNCTION void +eta_to_dp (const KernelVariables& kv, const int nlev, + const Real hy_ps0, const CRnV& hy_bi, const CRnV& hy_etai, + const CRelV& ps, const CRelnV& etai, const RelnV& wrk, + const RelnV& dp) { + const int nlevp = nlev + 1; + assert(hy_bi.extent_int(0) >= nlevp); + assert(hy_etai.extent_int(0) >= nlevp); + assert_eln(etai, nlevp); + assert_eln(wrk, nlevp); + assert_eln(dp, nlev); + const auto& bi = wrk; + const auto ttr = Kokkos::TeamThreadRange(kv.team, NP*NP); + const auto tvr_linterp = Kokkos::ThreadVectorRange(kv.team, nlevp); + const auto f_linterp = [&] (const int idx) { + const int i = idx / NP, j = idx % NP; + linterp(tvr_linterp, + nlevp, hy_etai, hy_bi, + nlevp, getcolc(etai,i,j), getcol(bi,i,j), + 0, "eta_to_dp"); + }; + Kokkos::parallel_for(ttr, f_linterp); + kv.team_barrier(); + const auto tvr = Kokkos::ThreadVectorRange(kv.team, nlev); + const auto f = [&] (const int idx) { + const int i = idx / NP, j = idx % NP; + const auto dps = ps(i,j) - hy_ps0; + const auto g = [&] (const int k) { + dp(i,j,k) = ((etai(i,j,k+1) - etai(i,j,k))*hy_ps0 + + (bi(i,j,k+1) - bi(i,j,k))*dps); + }; + Kokkos::parallel_for(tvr, g); + }; + Kokkos::parallel_for(ttr, f); +} + +/* Limit eta levels so their thicknesses, deta, are bounded below by 'low'. + + This method pulls mass only from intervals k that are larger than their + reference value (deta(k) > deta_ref(k)), and only down to their reference + value. This concentrates changes to intervals that, by having a lot more mass + than usual, drive other levels negative, leaving all the other intervals + unchanged. + + This selective use of mass provides enough to fulfill the needed mass. + Inputs: + m (deta): input mass + r (deta_ref): level mass reference. + Preconditions: + (1) 0 <= low <= min r(i) + (2) 1 = sum r(i) = sum(m(i)). + Rewrite (2) as + 1 = sum_{m(i) >= r(i)} m(i) + sum_{m(i) < r(i)} m(i) + and, thus, + 0 = sum_{m(i) >= r(i)} (m(i) - r(i)) + sum_{m(i) < r(i)} (m(i) - r(i)). + Then + sum_{m(i) >= r(i)} (m(i) - r(i)) (available mass to redistribute) + = -sum_{m(i) < r(i)} (m(i) - r(i)) + >= -sum_{m(i) < lo } (m(i) - r(i)) + >= -sum_{m(i) < lo } (m(i) - lo ) (mass to fill in). + Thus, if the preconditions hold, then there's enough mass to redistribute. + */ +template +KOKKOS_FUNCTION void +deta_caas (const KernelVariables& kv, const Range& tvr_nlevp, + const CRnV& deta_ref, const Real low, const RnV& w, + const RnV& deta) { + const auto g1 = [&] (const int k, Kokkos::Real2& sums) { + Real wk; + if (deta(k) < low) { + sums.v[0] += deta(k) - low; + deta(k) = low; + wk = 0; + } else { + wk = (deta(k) > deta_ref(k) ? + deta(k) - deta_ref(k) : + 0); + } + sums.v[1] += wk; + w(k) = wk; + }; + Kokkos::Real2 sums; + Dispatch<>::parallel_reduce(kv.team, tvr_nlevp, g1, sums); + const Real wneeded = sums.v[0]; + if (wneeded == 0) return; + // Remove what is needed from the donors. + const Real wavail = sums.v[1]; + const auto g2 = [&] (const int k) { + deta(k) += wneeded*(w(k)/wavail); + }; + Kokkos::parallel_for(tvr_nlevp, g2); +} + +// Wrapper to above. +KOKKOS_FUNCTION void +deta_caas (const KernelVariables& kv, const int nlevp, const CRnV& deta_ref, + const Real low, const RelnV& wrk, const RelnV& deta) { + assert(deta_ref.extent_int(0) >= nlevp); + assert_eln(wrk, nlevp); + assert_eln(deta, nlevp); + const auto ttr = Kokkos::TeamThreadRange(kv.team, NP*NP); + const auto tvr = Kokkos::ThreadVectorRange(kv.team, nlevp); + const auto f = [&] (const int idx) { + const int i = idx / NP, j = idx % NP; + deta_caas(kv, tvr, deta_ref, low, getcol(wrk,i,j), getcol(deta,i,j)); + }; + Kokkos::parallel_for(ttr, f); +} + +// Wrapper to deta_caas. On input and output, eta contains the midpoint eta +// values. On output, deta_caas has been applied, if necessary, to +// diff(eta(i,j,:)). +KOKKOS_FUNCTION void +limit_etam (const KernelVariables& kv, const int nlev, const CRnV& hy_etai, + const CRnV& deta_ref, const Real deta_tol, const RelnV& wrk1, + const RelnV& wrk2, const RelnV& eta) { + assert(hy_etai.extent_int(0) >= nlev+1); + assert(deta_ref.extent_int(0) >= nlev+1); + const auto deta = wrk2; + assert_eln(wrk1, nlev+1); + assert_eln(deta, nlev+1); + assert_eln(eta , nlev ); + const auto ttr = Kokkos::TeamThreadRange(kv.team, NP*NP); + const auto tvr = Kokkos::ThreadVectorRange(kv.team, nlev+1); + // eta -> deta; limit deta if needed. + const auto f1 = [&] (const int idx) { + const int i = idx / NP, j = idx % NP; + const auto etaij = getcolc( eta,i,j); + const auto detaij = getcol(deta,i,j); + const auto g1 = [&] (const int k, int& nbad) { + const auto d = (k == 0 ? etaij(0) - hy_etai(0) : + k == nlev ? hy_etai(nlev) - etaij(nlev-1) : + /**/ etaij(k) - etaij(k-1)); + const bool ok = d >= deta_tol; + if (not ok) ++nbad; + detaij(k) = d; + }; + int nbad = 0; + Dispatch<>::parallel_reduce(kv.team, tvr, g1, nbad); + if (nbad == 0) { + // Signal this column is fine. + Kokkos::single(Kokkos::PerThread(kv.team), [&] () { detaij(0) = -1; }); + return; + }; + deta_caas(kv, tvr, deta_ref, deta_tol, getcol(wrk1,i,j), detaij); + }; + Kokkos::parallel_for(ttr, f1); + kv.team_barrier(); + // deta -> eta; ignore columns where limiting wasn't needed. + const auto f2 = [&] (const int idx) { + const int i = idx / NP, j = idx % NP; + const auto etaij = getcol( eta,i,j); + const auto detaij = getcol(deta,i,j); + if (detaij(0) == -1) return; + const auto g = [&] (const int k, Real& accum, const bool final) { + assert(k != 0 or accum == 0); + const Real d = k == 0 ? hy_etai(0) + detaij(0) : detaij(k); + accum += d; + if (final) etaij(k) = accum; + }; + Dispatch<>::parallel_scan(kv.team, nlev, g); + }; + Kokkos::parallel_for(ttr, f2); +} + +// Compute surface pressure ps = ai(0) ps0 + sum dp. +KOKKOS_FUNCTION void calc_ps ( + const KernelVariables& kv, const int nlev, + const Real& ps0, const Real& hyai0, + const CSelnV& dp, + const ExecViewUnmanaged& ps) +{ + assert_eln(dp, nlev); + const auto ttr = Kokkos::TeamThreadRange(kv.team, NP*NP); + const auto tvr_snlev = Kokkos::ThreadVectorRange(kv.team, nlev); + const CRelnV dps = elp2r(dp); + const auto f1 = [&] (const int idx) { + const int i = idx / NP, j = idx % NP; + const auto g = [&] (int k, Real& sum) { sum += dps(i,j,k); }; + Real sum; + Dispatch<>::parallel_reduce(kv.team, tvr_snlev, g, sum); + Kokkos::single(Kokkos::PerThread(kv.team), + [&] { ps(i,j) = hyai0*ps0 + sum; }); + }; + Kokkos::parallel_for(ttr, f1); +} + +// Compute the surface pressure ps[i] at time point i corresponding to +// dp[i] = (1-alpha[i]) dp1 + alpha[i] dp2. +KOKKOS_FUNCTION void calc_ps ( + const KernelVariables& kv, const int nlev, + const Real& ps0, const Real& hyai0, + const Real alpha[2], const CSelnV& dp1, const CSelnV& dp2, + const ExecViewUnmanaged& ps) +{ + assert_eln(dp1, nlev); + assert_eln(dp2, nlev); + const auto ttr = Kokkos::TeamThreadRange(kv.team, NP*NP); + const auto tvr_snlev = Kokkos::ThreadVectorRange(kv.team, nlev); + const CRelnV dps[] = {elp2r(dp1), elp2r(dp2)}; + const auto f1 = [&] (const int idx) { + const int i = idx / NP, j = idx % NP; + for (int t = 0; t < 2; ++t) { + const auto& dp = dps[t]; + const auto g = [&] (int k, Real& sum) { sum += dp(i,j,k); }; + Real sum; + Dispatch<>::parallel_reduce(kv.team, tvr_snlev, g, sum); + Kokkos::single(Kokkos::PerThread(kv.team), [&] { ps(t,i,j) = sum; }); + } + }; + Kokkos::parallel_for(ttr, f1); + kv.team_barrier(); + const auto f2 = [&] (const int idx) { + const int i = idx / NP, j = idx % NP; + const auto g = [&] () { + Real vals[2]; + for (int t = 0; t < 2; ++t) + vals[t] = (hyai0*ps0 + + (1 - alpha[t])*ps(0,i,j) + + /**/ alpha[t] *ps(1,i,j)); + for (int t = 0; t < 2; ++t) + ps(t,i,j) = vals[t]; + }; + Kokkos::single(Kokkos::PerThread(kv.team), g); + }; + Kokkos::parallel_for(ttr, f2); +} + +// Transform eta_dot_dpdn at interfaces to eta_dot at midpoints using the +// formula +// eta_dot = eta_dot_dpdn/(A_eta p0 + B_eta ps). +// a= eta_dot_dpdn diff(eta)/(diff(A) p0 + diff(B) ps). +KOKKOS_FUNCTION void calc_etadotmid_from_etadotdpdnint ( + const KernelVariables& kv, const int nlev, + const Real& ps0, const CSnV& hydai, const CSnV& hydbi, + const CSnV& hydetai, const CRelV& ps, const SelnV& wrk, + // in: eta_dot_dpdn at interfaces + // out: eta_dot at midpoints, final slot unused + const SelnV& ed) +{ + assert(calc_nscal(hydai.extent_int(0)) >= nlev); + assert(calc_nscal(hydbi.extent_int(0)) >= nlev); + assert(calc_nscal(hydetai.extent_int(0)) >= nlev); + assert_eln(wrk, nlev+1); + assert_eln(ed, nlev+1); + const auto& edd_mid = wrk; + { + const CRelnV edd(elp2r(ed)); + const RelnV tmp(elp2r(wrk)); + const auto f = [&] (const int i, const int j, const int k) { + tmp(i,j,k) = (edd(i,j,k) + edd(i,j,k+1))/2; + }; + cti::loop_ijk(nlev, kv, f); + } + kv.team_barrier(); + { + const auto f = [&] (const int i, const int j, const int kp) { + ed(i,j,kp) = (edd_mid(i,j,kp) + * hydetai(kp) + / (hydai(kp)*ps0 + hydbi(kp)*ps(i,j))); + }; + cti::loop_ijk(calc_npack(nlev), kv, f); + } +} + +// Compute eta_dot at midpoint nodes at the start and end of the substep. +KOKKOS_FUNCTION void calc_eta_dot_ref_mid ( + const KernelVariables& kv, const SphereOperators& sphere_ops, + const Real& ps0, const Real& hyai0, const CSNV& hybi, + const CSNV& hydai, const CSNV& hydbi, // delta ai, bi + const CSNV& hydetai, // delta etai + const Real alpha[2], + const CS2elNlev& v1, const CSelNlev& dp1, const CS2elNlev& v2, const CSelNlev& dp2, + const SelNlevp& wrk1, const SelNlevp& wrk2, const S2elNlevp& vwrk1, + // Holds interface levels as intermediate data but is midpoint data on output, + // with final slot unused. + const SelNlevp eta_dot[2]) +{ + using Kokkos::ALL; + const int nlev = NUM_PHYSICAL_LEV; + const SelNlev divdp(wrk1.data()); + const S2elNlev vdp(vwrk1.data()); + const ExecViewUnmanaged ps(cti::pack2real(wrk2)); + // Calc surface pressure for use at the end. + calc_ps(kv, nlev, + ps0, hyai0, + alpha, dp1, dp2, + ps); + kv.team_barrier(); + for (int t = 0; t < 2; ++t) { + // Compute divdp. + const auto f = [&] (const int i, const int j, const int kp) { + for (int d = 0; d < 2; ++d) + vdp(d,i,j,kp) = ((1 - alpha[t])*v1(d,i,j,kp)*dp1(i,j,kp) + + /**/ alpha[t] *v2(d,i,j,kp)*dp2(i,j,kp)); + }; + cti::loop_ijk(kv, f); + kv.team_barrier(); + sphere_ops.divergence_sphere(kv, vdp, divdp); + kv.team_barrier(); + // Compute eta_dot_dpdn at interface nodes. + const auto& edd = eta_dot[t]; + const RelNlevp edds(cti::pack2real(edd)); + const RelNlev divdps(cti::pack2real(wrk1)); + cti::calc_eta_dot_dpdn(kv, + hybi, + divdps, edd, + edds); + kv.team_barrier(); + calc_etadotmid_from_etadotdpdnint(kv, nlev, + ps0, hydai, hydbi, hydetai, + Kokkos::subview(ps,t,ALL,ALL), + wrk1, + edd); + // No team_barrier: wrk1 is protected in second iteration. + } +} + +// Given the vertical and horizontal nodal velocities at time endpoints, +// evaluate the velocity estimate formula, providing the final horizontal +// velocity estimates at midpoint nodes. +KOKKOS_FUNCTION void calc_vel_horiz_formula_node_ref_mid ( + const KernelVariables& kv, const SphereOperators& sphere_ops, + const CSNV& hyetam, const ExecViewUnmanaged& vec_sph2cart, + // Velocities are at midpoints. Final eta_dot entry is ignored. + const Real dtsub, const CS2elNlev vsph[2], const CSelNlevp eta_dot[2], + const SelNlevp& wrk1, const S2elNlevp& vwrk1, const S2elNlevp& vwrk2, + const ExecViewUnmanaged& vnode) +{ + using Kokkos::ALL; + const S2elNlev vfsph(vwrk1.data()), vw2(vwrk2.data()); + const SelNlev w1(wrk1.data()); + const R2elNlev vfsphs(cti::pack2real(vfsph)); + const auto& vsph1 = vsph[0]; + const auto& vsph2 = vsph[1]; + { // Horizontal terms. + cti::ugradv_sphere(sphere_ops, kv, vec_sph2cart, vsph2, vsph1, w1, vw2, vfsph); + for (int d = 0; d < 2; ++d) { + const auto f = [&] (const int i, const int j, const int k) { + vfsph(d,i,j,k) = vsph1(d,i,j,k) + vsph2(d,i,j,k) - dtsub*vfsph(d,i,j,k); + }; + cti::loop_ijk(kv, f); + } + } + kv.team_barrier(); + { // Vertical terms. + const CRNV etams(cti::cpack2real(hyetam)); + const CR2elNlev vsph1s(cti::cpack2real(vsph1)); + const CRelNlevp eds(cti::cpack2real(eta_dot[1])); + for (int d = 0; d < 2; ++d) { + const auto f = [&] (const int i, const int j, const int k) { + Real deriv; + if (k == 0 or k+1 == NUM_PHYSICAL_LEV) { + const int k1 = k == 0 ? 0 : NUM_PHYSICAL_LEV-2; + const int k2 = k == 0 ? 1 : NUM_PHYSICAL_LEV-1; + deriv = ((vsph1s(d,i,j,k2) - vsph1s(d,i,j,k1)) / + (etams(k2) - etams(k1))); + } else { + deriv = cti::approx_derivative( + etams(k-1), etams(k), etams(k+1), + vsph1s(d,i,j,k-1), vsph1s(d,i,j,k), vsph1s(d,i,j,k+1)); + } + vfsphs(d,i,j,k) = (vfsphs(d,i,j,k) - dtsub*eds(i,j,k)*deriv)/2; + }; + cti::loop_ijk(kv, f); + } + } + { // Transform to Cartesian. + for (int d = 0; d < 3; ++d) { + const auto f = [&] (const int i, const int j, const int k) { + vnode(k,i,j,d) = (vec_sph2cart(0,d,i,j)*vfsphs(0,i,j,k) + + vec_sph2cart(1,d,i,j)*vfsphs(1,i,j,k)); + }; + cti::loop_ijk(kv, f); + } + } +} + +// Given the vertical and horizontal nodal velocities at time endpoints, +// evaluate the velocity estimate formula, providing the final vertical velocity +// estimates at midpoint nodes. +KOKKOS_FUNCTION void calc_eta_dot_formula_node_ref_mid ( + const KernelVariables& kv, const SphereOperators& sphere_ops, + const CRNV& hyetai, const CSNV& hyetam, + // Velocities are at midpoints. Final eta_dot entry is ignored. + const Real dtsub, const CS2elNlev vsph[2], const CSelNlevp eta_dot[2], + const SelNlevp& wrk1, const S2elNlevp& vwrk1, + const ExecViewUnmanaged& vnode) +{ + const SelNlev ed1_vderiv(wrk1.data()); + { + const CRNV etams(cti::cpack2real(hyetam)); + const CRelNlevp ed1s(cti::cpack2real(eta_dot[0])); + const RelNlev ed1_vderiv_s(cti::pack2real(ed1_vderiv)); + const auto f = [&] (const int i, const int j, const int k) { + Real deriv; + if (k == 0 or k+1 == NUM_PHYSICAL_LEV) { + deriv = cti::approx_derivative( + k == 0 ? hyetai(0) : etams(k-1), + etams(k), + k+1 == NUM_PHYSICAL_LEV ? hyetai(NUM_PHYSICAL_LEV) : etams(k+1), + k == 0 ? 0 : ed1s(i,j,k-1), + ed1s(i,j,k), + k+1 == NUM_PHYSICAL_LEV ? 0 : ed1s(i,j,k+1)); + } else { + deriv = cti::approx_derivative( + etams(k-1), etams(k), etams(k+1), + ed1s(i,j,k-1), ed1s(i,j,k), ed1s(i,j,k+1)); + } + ed1_vderiv_s(i,j,k) = deriv; + }; + cti::loop_ijk(kv, f); + } + kv.team_barrier(); + const S2elNlev ed1_hderiv(vwrk1.data()); + sphere_ops.gradient_sphere(kv, eta_dot[0], ed1_hderiv, NUM_LEV); + { + const auto& vsph2 = vsph[1]; + const auto& ed1 = eta_dot[0]; + const auto& ed2 = eta_dot[1]; + const auto f = [&] (const int i, const int j, const int k) { + const auto v = (ed1(i,j,k) + ed2(i,j,k) + - dtsub*( vsph2(0,i,j,k)*ed1_hderiv(0,i,j,k) + + vsph2(1,i,j,k)*ed1_hderiv(1,i,j,k) + + ed2( i,j,k)*ed1_vderiv( i,j,k)))/2; + for (int s = 0; s < VECTOR_SIZE; ++s) + vnode(VECTOR_SIZE*k+s, i,j,3) = v[s]; + }; + cti::loop_ijk(kv, f); + } +} + +} // namespace anon +} // namespace Homme + +#endif +#endif diff --git a/components/homme/src/share/cxx/ComposeTransportImplEnhancedTrajectoryTests.cpp b/components/homme/src/share/cxx/ComposeTransportImplEnhancedTrajectoryTests.cpp new file mode 100644 index 000000000000..da8fbab4d919 --- /dev/null +++ b/components/homme/src/share/cxx/ComposeTransportImplEnhancedTrajectoryTests.cpp @@ -0,0 +1,839 @@ +/******************************************************************************** + * HOMMEXX 1.0: Copyright of Sandia Corporation + * This software is released under the BSD license + * See the file 'COPYRIGHT' in the HOMMEXX/src/share/cxx directory + *******************************************************************************/ + +#include "Config.hpp" +#ifdef HOMME_ENABLE_COMPOSE + +#include "ComposeTransportImplEnhancedTrajectoryImpl.hpp" + +#include + +namespace Homme { +namespace { // anon + +Kokkos::TeamPolicy +get_test_team_policy (const int nelem, const int nlev, const int ncol=NP*NP) { + ThreadPreferences tp; + tp.max_threads_usable = ncol; + tp.max_vectors_usable = nlev; + tp.prefer_threads = true; + tp.prefer_larger_team = true; + return Homme::get_default_team_policy(nelem, tp); +} + +struct TestData { + std::mt19937_64 engine; + static const Real eps; + const ComposeTransportImpl& cti; + + TestData (const CTI& cti_, const int seed = 0) + : cti(cti_), engine(seed == 0 ? std::random_device()() : seed) + {} + + Real urand (const Real lo = 0, const Real hi = 1) { + std::uniform_real_distribution urb(lo, hi); + return urb(engine); + } +}; + +// Data to deal with views of packs easily in tests. +struct ColData { + int npack; + ExecView d; + ExecView::HostMirror h; + ExecView::HostMirror r; + + ColData (const std::string& name, const int nlev) { + npack = calc_npack(nlev); + d = decltype(d)(name, npack); + h = Kokkos::create_mirror_view(d); + r = decltype(r)(cti::pack2real(h), calc_nscal(npack)); + } + + void h2d () { Kokkos::deep_copy(d, h); } +}; + +struct ElData { + int npack; + ExecView d; + ExecView::HostMirror h; + ExecView::HostMirror r; + + ElData (const std::string& name, const int nlev) { + npack = calc_npack(nlev); + d = decltype(d)(name, NP, NP, npack); + h = Kokkos::create_mirror_view(d); + r = decltype(r)(cti::pack2real(h), NP, NP, calc_nscal(npack)); + } + + void d2h () { Kokkos::deep_copy(h, d); } + void h2d () { Kokkos::deep_copy(d, h); } +}; + +const Real TestData::eps = std::numeric_limits::epsilon(); + +int test_find_support (TestData&) { + int ne = 0; + const int n = 97; + std::vector x(n); + for (int i = 0; i < n; ++i) x[i] = -11.7 + (i*i)/n; + const int ntest = 10000; + for (int i = 0; i < ntest; ++i) { + const Real xi = x[0] + (Real(i)/ntest)*(x[n-1] - x[0]); + for (int x_idx : {0, 1, n/3, n/2, n-2, n-1}) { + const int sup = find_support(n, x.data(), x_idx, xi); + if (sup > n-2) ++ne; + else if (xi < x[sup] or xi > x[sup+1]) ++ne; + } + } + return ne; +} + +void todev (const std::vector& h, const RnV& d) { + assert(h.size() <= d.size()); + const auto m = Kokkos::create_mirror_view(d); + for (size_t i = 0; i < h.size(); ++i) m(i) = h[i]; + Kokkos::deep_copy(d, m); +} + +void fillcols (const int n, const Real* const h, const RelnV::HostMirror& a) { + assert(n <= a.extent_int(2)); + for (int i = 0; i < a.extent_int(0); ++i) + for (int j = 0; j < a.extent_int(1); ++j) + for (size_t k = 0; k < n; ++k) + a(i,j,k) = h[k]; +} + +void todev (const int n, const Real* const h, const RelnV& d) { + const auto m = Kokkos::create_mirror_view(d); + fillcols(n, h, m) ; + Kokkos::deep_copy(d, m); +} + +void todev (const std::vector& h, const RelnV& d) { + todev(h.size(), h.data(), d); +} + +void tohost (const ExecView& d, std::vector& h) { + assert(h.size() <= d.size()); + const auto m = Kokkos::create_mirror_view(d); + Kokkos::deep_copy(m, d); + for (size_t i = 0; i < h.size(); ++i) h[i] = m(i); +} + +void run_linterp (const std::vector& x, const std::vector& y, + std::vector& xi, std::vector& yi) { + const auto n = x.size(), ni = xi.size(); + assert(y.size() == n); assert(yi.size() == ni); + // input -> device (test different sizes >= n) + ExecView xv("xv", n), yv("yv", n+1), xiv("xiv", ni+2), yiv("yiv", ni+3); + todev(x, xv); + todev(y, yv); + todev(xi, xiv); + // call linterp + const auto f = KOKKOS_LAMBDA(const cti::MT& team) { + const auto range = Kokkos::TeamVectorRange(team, ni); + linterp(range, n, xv, yv, ni, xiv, yiv, 0, "unittest"); + }; + Homme::ThreadPreferences tp; + tp.max_threads_usable = 1; + tp.max_vectors_usable = ni; + tp.prefer_threads = false; + tp.prefer_larger_team = true; + const auto policy = get_test_team_policy(1, n); + Kokkos::parallel_for(policy, f); + Kokkos::fence(); + // output -> host + tohost(yiv, yi); +} + +void make_random_sorted (TestData& td, const int n, const Real xlo, const Real xhi, + std::vector& x) { + assert(n >= 2); + x.resize(n); + x[0] = xlo; + for (int i = 1; i < n-1; ++i) x[i] = td.urand(xlo, xhi); + x[n-1] = xhi; + std::sort(x.begin(), x.end()); +} + +int test_linterp (TestData& td) { + int nerr = 0; + { // xi == x => yi == y. + int ne = 0; + const int n = 30; + std::vector x(n), y(n), xi(n), yi(n); + make_random_sorted(td, n, -0.1, 1.2, x); + make_random_sorted(td, n, -3, -1, y); + for (int i = 0; i < n; ++i) xi[i] = x[i]; + run_linterp(x, y, xi, yi); + for (int i = 0; i < n; ++i) + if (yi[i] != y[i]) + ++ne; + nerr += ne; + } + { // Reconstruct a linear function exactly. + int ne = 0; + const int n = 56, ni = n-3; + const Real xlo = -1.2, xhi = 3.1; + const auto f = [&] (const Real x) { return -0.7 + 1.3*x; }; + std::vector x(n), y(n), xi(ni), yi(ni); + for (int trial = 0; trial < 4; ++trial) { + make_random_sorted(td, n, xlo, xhi, x); + make_random_sorted(td, ni, + xlo + (trial == 1 or trial == 3 ? 0.1 : 0), + xhi + (trial == 2 or trial == 3 ? -0.5 : 0), + xi); + for (int i = 0; i < n; ++i) y[i] = f(x[i]); + run_linterp(x, y, xi, yi); + for (int i = 0; i < ni; ++i) + if (std::abs(yi[i] - f(xi[i])) > 100*td.eps) + ++ne; + } + nerr += ne; + } + return nerr; +} + +int make_random_deta (TestData& td, const Real deta_tol, const int nlev, + Real* const deta) { + int nerr = 0; + Real sum = 0; + for (int k = 0; k < nlev; ++k) { + deta[k] = td.urand(0, 1) + 0.1; + sum += deta[k]; + } + for (int k = 0; k < nlev; ++k) { + deta[k] /= sum; + if (deta[k] < deta_tol) ++nerr; + } + return nerr; +} + +int make_random_deta (TestData& td, const Real deta_tol, const RnV& deta) { + int nerr = 0; + const int nlev = deta.extent_int(0); + const auto m = Kokkos::create_mirror_view(deta); + nerr = make_random_deta(td, deta_tol, nlev, &m(0)); + Kokkos::deep_copy(deta, m); + return nerr; +} + +int make_random_deta (TestData& td, const Real deta_tol, const RelnV& deta) { + int nerr = 0; + const int nlev = deta.extent_int(2); + const auto m = Kokkos::create_mirror_view(deta); + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) + nerr += make_random_deta(td, deta_tol, nlev, &m(i,j,0)); + Kokkos::deep_copy(deta, m); + return nerr; +} + +int test_deta_caas (TestData& td) { + int nerr = 0; + const Real tol = 100*td.eps; + + for (const int nlev : {15, 128, 161}) { + const Real deta_tol = 10*td.eps/nlev; + const auto err = [&] (const char* lbl) { + ++nerr; + printf("test_deta_caa nlev %d: %s\n", nlev, lbl); + }; + + // nlev+1 deltas: deta = diff([0, etam, 1]) + ExecView deta_ref("deta_ref", nlev+1); + ExecView deta("deta",NP,NP,nlev+1), wrk("wrk",NP,NP,nlev+1); + nerr += make_random_deta(td, deta_tol, deta_ref); + + const auto policy = get_test_team_policy(1, nlev); + const auto run = [&] (const RelnV& deta) { + const auto f = KOKKOS_LAMBDA(const cti::MT& team) { + KernelVariables kv(team); + deta_caas(kv, nlev+1, deta_ref, deta_tol, wrk, deta); + }; + Kokkos::parallel_for(policy, f); + Kokkos::fence(); + }; + + { // Test that if all is OK, the input is not altered. + nerr += make_random_deta(td, deta_tol, deta); + ExecView::HostMirror copy("copy",NP,NP,nlev+1); + Kokkos::deep_copy(copy, deta); + run(deta); + const auto m = cti::cmvdc(deta); + bool diff = false; + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) + for (int k = 0; k <= nlev; ++k) + if (m(i,j,k) != copy(i,j,k)) + diff = true; + if (diff) err("input not altered"); + } + + { // Modify one etam and test that only adjacent intervals change beyond eps. + // nlev midpoints + ExecView etam_ref("etam_ref",nlev); + const auto her = Kokkos::create_mirror_view(etam_ref); + const auto hder = cti::cmvdc(deta_ref); + { + her(0) = hder(0); + for (int k = 1; k < nlev; ++k) + her(k) = her(k-1) + hder(k); + Kokkos::deep_copy(etam_ref, her); + } + std::vector etam(nlev); + const auto hde = Kokkos::create_mirror_view(deta); + const auto get_idx = [&] (const int i, const int j) { + const int idx = static_cast(0.15*nlev); + return std::max(1, std::min(nlev-2, idx+NP*i+j)); + }; + for (int trial = 0; trial < 2; ++trial) { + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) { + for (int k = 0; k < nlev; ++k) etam[k] = her(k); + // Perturb one level. + const int idx = get_idx(i,j); + etam[idx] += trial == 0 ? 1.1 : -13.1; + hde(i,j,0) = etam[0]; + for (int k = 1; k < nlev; ++k) hde(i,j,k) = etam[k] - etam[k-1]; + hde(i,j,nlev) = 1 - etam[nlev-1]; + // Make sure we have a meaningful test. + Real minval = 1; + for (int k = 0; k <= nlev; ++k) minval = std::min(minval, hde(i,j,k)); + if (minval >= deta_tol) err("meaningful test"); + } + Kokkos::deep_copy(deta, hde); + run(deta); + Kokkos::deep_copy(hde, deta); + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) { + const int idx = get_idx(i,j); + // Min val should be deta_tol. + Real minval = 1; + for (int k = 0; k <= nlev; ++k) minval = std::min(minval, hde(i,j,k)); + if (minval != deta_tol) err("min val"); + // Sum of levels should be 1. + Real sum = 0; + for (int k = 0; k <= nlev; ++k) sum += hde(i,j,k); + if (std::abs(sum - 1) > tol) err("sum 1"); + // Only two deltas should be affected. + Real maxdiff = 0; + for (int k = 0; k <= nlev; ++k) { + const auto diff = std::abs(hde(i,j,k) - hder(k)); + if (k == idx or k == idx+1) { + if (diff <= deta_tol) err("2 deltas a"); + } else { + maxdiff = std::max(maxdiff, diff); + } + } + if (maxdiff > tol) err("2 deltas b"); + } + } + } + + { // Test generally (and highly) perturbed levels. + const auto hde = Kokkos::create_mirror_view(deta); + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) { + Real sum = 0; + for (int k = 0; k <= nlev; ++k) { + hde(i,j,k) = td.urand(-0.5, 0.5); + sum += hde(i,j,k); + } + // Make the column sum to 0.2 for safety in the next step. + const Real colsum = 0.2; + for (int k = 0; k <= nlev; ++k) hde(i,j,k) += (colsum - sum)/(nlev+1); + for (int k = 0; k <= nlev; ++k) hde(i,j,k) /= colsum; + sum = 0; + for (int k = 0; k <= nlev; ++k) sum += hde(i,j,k); + if (std::abs(sum - 1) > 10*tol) err("general sum 1"); + } + Kokkos::deep_copy(deta, hde); + run(deta); + Kokkos::deep_copy(hde, deta); + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) { + Real sum = 0, minval = 1; + for (int k = 0; k <= nlev; ++k) sum += hde(i,j,k); + for (int k = 0; k <= nlev; ++k) minval = std::min(minval, hde(i,j,k)); + if (std::abs(sum - 1) > 1e3*td.eps) ++nerr; + if (minval != deta_tol) err("general minval"); + } + } + } + + return nerr; +} + +struct HybridLevels { + Real ps0, a_eta, b_eta; + std::vector ai, dai, bi, dbi, am, bm, etai, detai, etam, detam; +}; + +// Follow DCMIP2012 3D tracer transport specification for a, b, eta. +void fill (HybridLevels& h, const int n) { + h.ai.resize(n+1); h.bi.resize(n+1); + h.am.resize(n ); h.bm.resize(n ); + h.etai.resize(n+1); h.etam.resize(n); + + const auto Rd = PhysicalConstants::Rgas; + const auto T0 = 300; // K + const auto p0 = PhysicalConstants::p0; + const auto g = PhysicalConstants::g; + const Real ztop = 12e3; // m + + h.ps0 = p0; + + const auto calc_pressure = [&] (const Real z) { + return p0*std::exp(-g*z/(Rd*T0)); + }; + + const Real eta_top = calc_pressure(ztop)/p0; + assert(eta_top > 0); + for (int i = 0; i <= n; ++i) { + const auto z = (Real(n - i)/n)*ztop; + h.etai[i] = calc_pressure(z)/p0; + h.bi[i] = i == 0 ? 0 : (h.etai[i] - eta_top)/(1 - eta_top); + h.ai[i] = h.etai[i] - h.bi[i]; + assert(i == 0 or h.etai[i] > h.etai[i-1]); + } + assert(h.bi [0] == 0); // Real(n - i)/n is exactly 1, so exact = holds + assert(h.bi [n] == 1); // exp(0) is exactly 0, so exact = holds + assert(h.etai[n] == 1); // same + // b = (eta - eta_top)/(1 - eta_top) => b_eta = 1/(1 - eta_top) + // a = eta - b => a_eta = 1 - b_eta = -eta_top/(1 - eta_top) + // p_eta = a_eta p0 + b_eta ps + h.b_eta = 1/(1 - eta_top); + h.a_eta = 1 - h.b_eta; + + const auto tomid = [&] (const std::vector& in, std::vector& mi) { + for (int i = 0; i < n; ++i) mi[i] = (in[i] + in[i+1])/2; + }; + tomid(h.ai, h.am); + tomid(h.bi, h.bm); + tomid(h.etai, h.etam); + + const auto diff = [&] (const std::vector& ai, std::vector& dai) { + dai.resize(n); + for (int i = 0; i < n; ++i) dai[i] = ai[i+1] - ai[i]; + }; + diff(h.ai, h.dai); + diff(h.bi, h.dbi); + diff(h.etai, h.detai); + + h.detam.resize(n+1); + h.detam[0] = h.etam[0] - h.etai[0]; + for (int i = 1; i < n; ++i) h.detam[i] = h.etam[i] - h.etam[i-1]; + h.detam[n] = h.etai[n] - h.etam[n-1]; +} + +int test_limit_etam (TestData& td) { + int nerr = 0; + const Real tol = 100*td.eps; + + for (const int nlev : {143, 128, 81}) { + const Real deta_tol = 1e5*td.eps/nlev; + + ExecView hy_etai("hy_etai",nlev+1), detam("detam",nlev+1); + ExecView wrk1("wrk1",NP,NP,nlev+1), wrk2("wrk2",NP,NP,nlev+1); + ExecView etam("etam",NP,NP,nlev); + + HybridLevels h; + fill(h, nlev); + todev(h.etai, hy_etai); + todev(h.detam, detam); + + const auto he = Kokkos::create_mirror_view(etam); + + const auto policy = get_test_team_policy(1, nlev); + const auto run = [&] () { + Kokkos::deep_copy(etam, he); + const auto f = KOKKOS_LAMBDA(const cti::MT& team) { + KernelVariables kv(team); + limit_etam(kv, nlev, hy_etai, detam, deta_tol, wrk1, wrk2, etam); + }; + Kokkos::parallel_for(policy, f); + Kokkos::fence(); + Kokkos::deep_copy(he, etam); + }; + + fillcols(h.etam.size(), h.etam.data(), he); + // Col 0 should be untouched. Cols 1 and 2 should have very specific changes. + const int col1_idx = static_cast(0.25*nlev); + he(0,1,col1_idx) += 0.3; + const int col2_idx = static_cast(0.8*nlev); + he(0,2,col2_idx) -= 5.3; + // The rest of the columns get wild changes. + for (int idx = 3; idx < NP*NP; ++idx) { + const int i = idx / NP, j = idx % NP; + for (int k = 0; k < nlev; ++k) + he(i,j,k) += td.urand(-1, 1)*(h.etai[k+1] - h.etai[k]); + } + run(); + bool ok = true; + for (int k = 0; k < nlev; ++k) + if (he(0,0,k) != h.etam[k]) ok = false; + for (int k = 0; k < nlev; ++k) { + if (k == col1_idx) continue; + if (std::abs(he(0,1,k) - h.etam[k]) > tol) ok = false; + } + for (int k = 0; k < nlev; ++k) { + if (k == col2_idx) continue; + if (std::abs(he(0,2,k) - h.etam[k]) > tol) ok = false; + } + Real mingap = 1; + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) { + mingap = std::min(mingap, he(i,j,0) - h.etai[0]); + for (int k = 1; k < nlev; ++k) + mingap = std::min(mingap, he(i,j,k) - he(i,j,k-1)); + mingap = std::min(mingap, h.etai[nlev] - he(i,j,nlev-1)); + } + // Test minimum level delta, with room for numerical error. + if (mingap < 0.8*deta_tol) ok = false; + if (not ok) ++nerr; + } + + return nerr; +} + +int test_eta_interp (TestData& td) { + int nerr = 0; + const Real tol = 100*td.eps; + + for (const int nlev : {15, 128, 161}) { + HybridLevels h; + fill(h, nlev); + + ExecView hy_etai("hy_etai",nlev+1); + ExecView x("x",NP,NP,nlev), y("y",NP,NP,nlev); + ExecView xi("xi",NP,NP,nlev+1), yi("yi",NP,NP,nlev+1); + ExecView xwrk("xwrk",NP,NP,nlev+2), ywrk("ywrk",NP,NP,nlev+2); + + todev(h.etai, hy_etai); + + const auto xh = Kokkos::create_mirror_view(x ); + const auto yh = Kokkos::create_mirror_view(y ); + const auto xih = Kokkos::create_mirror_view(xi); + const auto yih = Kokkos::create_mirror_view(yi); + + const auto policy = get_test_team_policy(1, nlev); + const auto run_eta = [&] (const int ni) { + Kokkos::deep_copy(x, xh); Kokkos::deep_copy(y, yh); + Kokkos::deep_copy(xi, xih); + const auto f = KOKKOS_LAMBDA(const cti::MT& team) { + KernelVariables kv(team); + eta_interp_eta(kv, nlev, hy_etai, + x, getcolc(y,0,0), + xwrk, getcol(ywrk,0,0), + ni, getcolc(xi,0,0), yi); + }; + Kokkos::parallel_for(policy, f); + Kokkos::fence(); + Kokkos::deep_copy(yih, yi); + }; + const auto run_horiz = [&] () { + Kokkos::deep_copy(x, xh); Kokkos::deep_copy(y, yh); + Kokkos::deep_copy(xi, xih); + const auto f = KOKKOS_LAMBDA(const cti::MT& team) { + KernelVariables kv(team); + eta_interp_horiz(kv, nlev, hy_etai, + getcolc(x,0,0), y, + getcol(xwrk,0,0), ywrk, + xi, yi); + }; + Kokkos::parallel_for(policy, f); + Kokkos::fence(); + Kokkos::deep_copy(yih, yi); + }; + + std::vector v; + const Real d = 1e-6, vlo = h.etai[0]+d, vhi = h.etai[nlev]-d; + + for (const int ni : {int(0.7*nlev), nlev-1, nlev, nlev+1}) { + make_random_sorted(td, nlev, vlo, vhi, v); + fillcols(nlev, v.data(), xh); + fillcols(nlev, v.data(), yh); + make_random_sorted(td, ni, vlo, vhi, v); + fillcols(ni, v.data(), xih); + run_eta(ni); + bool ok = true; + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) + for (int k = 0; k < ni; ++k) + if (std::abs(yih(i,j,k) - xih(i,j,k)) > tol) + ok = false; + if (not ok) ++nerr; + } + + { // Test exact interp of line in the interior, const interp near the bdys. + make_random_sorted(td, nlev, vlo+0.05, vhi-0.1, v); + fillcols(nlev, v.data(), xh); + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) { + for (int k = 0; k < nlev; ++k) + yh(i,j,k) = i*xh(0,0,k) - j; + make_random_sorted(td, nlev, vlo, vhi, v); + for (int k = 0; k < nlev; ++k) + xih(i,j,k) = v[k]; + } + run_horiz(); + bool ok = true; + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) + for (int k = 0; k < nlev; ++k) { + if (xih(i,j,k) < xh(0,0,0)) { + if (std::abs(yih(i,j,k) - yih(i,j,0)) > tol) + ok = false; + } else if (xih(i,j,k) > xh(0,0,nlev-1)) { + if (std::abs(yih(i,j,k) - yih(i,j,nlev-1)) > tol) + ok = false; + } else { + if (std::abs(yih(i,j,k) - (i*xih(i,j,k) - j)) > tol) + ok = false; + } + } + if (not ok) ++nerr; + } + } + + return nerr; +} + +int test_eta_to_dp (TestData& td) { + int nerr = 0; + const Real tol = 100*td.eps; + + for (const int nlev : {143, 128, 81}) { + const auto err = [&] (const char* lbl) { + ++nerr; + printf("test_eta_to_dp nlev %d: %s\n", nlev, lbl); + }; + + HybridLevels h; + fill(h, nlev); + + ExecView hy_bi("hy_bi",nlev+1), hy_etai("hy_etai",nlev+1); + ExecView etai("etai",NP,NP,nlev+1), wrk("wrk",NP,NP,nlev+1); + ExecView dp("dp",NP,NP,nlev); + ExecView ps("ps"); + const Real hy_ps0 = h.ps0; + + todev(h.bi, hy_bi); + todev(h.etai, hy_etai); + + const auto psm = Kokkos::create_mirror_view(ps); + HostView dp1("dp1",NP,NP,nlev); + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) + psm(i,j) = (1 + 0.1*td.urand(-1, 1))*h.ps0; + Kokkos::deep_copy(ps, psm); + + const auto policy = get_test_team_policy(1, nlev); + const auto run = [&] () { + const auto f = KOKKOS_LAMBDA(const cti::MT& team) { + KernelVariables kv(team); + eta_to_dp(kv, nlev, hy_ps0, hy_bi, hy_etai, ps, etai, wrk, dp); + }; + Kokkos::parallel_for(policy, f); + Kokkos::fence(); + }; + + { // Test that for etai_ref we get the same as the usual formula. + todev(h.etai, etai); + HostView dp1("dp1",NP,NP,nlev); + Real dp1_max = 0; + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) + for (int k = 0; k < nlev; ++k) { + dp1(i,j,k) = ((h.ai[k+1] - h.ai[k])*h.ps0 + + (h.bi[k+1] - h.bi[k])*psm(i,j)); + dp1_max = std::max(dp1_max, std::abs(dp1(i,j,k))); + } + run(); + const auto dph = cti::cmvdc(dp); + Real err_max = 0; + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) + for (int k = 0; k < nlev; ++k) + err_max = std::max(err_max, std::abs(dph(i,j,k) - dp1(i,j,k))); + if (err_max > tol*dp1_max) err("t1"); + } + + { // Test that sum(dp) = ps for random input etai. + std::vector etai_r; + make_random_sorted(td, nlev+1, h.etai[0], h.etai[nlev], etai_r); + todev(etai_r, etai); + run(); + const auto dph1 = cti::cmvdc(dp); + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) { + Real ps = h.ai[0]*h.ps0; + for (int k = 0; k < nlev; ++k) + ps += dph1(i,j,k); + if (std::abs(ps - psm(i,j)) > tol*psm(i,j)) err("t2"); + } + // Test that values on input don't affect solution. + Kokkos::deep_copy(wrk, 0); + Kokkos::deep_copy(dp, 0); + run(); + const auto dph2 = cti::cmvdc(dp); + bool alleq = true; + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) + for (int k = 0; k < nlev; ++k) + if (dph2(i,j,k) != dph1(i,j,k)) + alleq = false; + if (not alleq) err("t3"); + } + } + + return nerr; +} + +int test_calc_ps (TestData& td) { + int nerr = 0; + const Real tol = 100*td.eps; + + for (const int nlev : {15, 128, 161}) { + HybridLevels h; + fill(h, nlev); + const auto ps0 = h.ps0, hyai0 = h.ai[0]; + + ElData dp1("dp1", nlev), dp2("dp2", nlev); + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) + for (int k = 0; k < nlev; ++k) { + dp1.r(i,j,k) = td.urand(0, 1000); + dp2.r(i,j,k) = td.urand(0, 1000); + } + dp1.h2d(); + dp2.h2d(); + + const Real alpha[] = {td.urand(0,1), td.urand(0,1)}; + + ExecView ps("ps"); + ExecView ps2("ps2"); + const auto policy = get_test_team_policy(1, nlev); + const auto f = KOKKOS_LAMBDA(const cti::MT& team) { + KernelVariables kv(team); + calc_ps(kv, nlev, ps0, hyai0, alpha, dp1.d, dp2.d, ps2); + calc_ps(kv, nlev, ps0, hyai0, dp1.d, ps); + }; + Kokkos::parallel_for(policy, f); + Kokkos::fence(); + + const auto ps_h = cti::cmvdc(ps); + const auto ps2_h = cti::cmvdc(ps2); + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) { + { + Real ps = h.ai[0]*h.ps0; + for (int k = 0; k < nlev; ++k) + ps += dp1.r(i,j,k); + if (std::abs(ps_h(i,j) - ps) > tol*ps) ++nerr; + } + for (int t = 0; t < 2; ++t) { + Real ps = h.ai[0]*h.ps0; + for (int k = 0; k < nlev; ++k) + ps += (1 - alpha[t])*dp1.r(i,j,k) + alpha[t]*dp2.r(i,j,k); + if (std::abs(ps2_h(t,i,j) - ps) > tol*ps) ++nerr; + } + } + } + + return nerr; +} + +int test_calc_etadotmid_from_etadotdpdnint (TestData& td) { + int nerr = 0; + const Real tol = 100*td.eps; + + for (const int nlev : {143, 128, 81}) { + HybridLevels h; + fill(h, nlev); + + // Test function: + // eta_dot_dpdn(eta) = c eta + d. + // Then + // eta_dot = eta_dot_dpdn(eta)/dpdn(eta) + // = (c eta + d)/(a_eta p0 + b_eta ps). + // Since a_eta, b_eta are constants independent of eta in this test, eta_dot + // is then also a linear function of eta. Thus, we can test for exact + // agreement with the true solution. + + ColData hydai("hydai",nlev), hydbi("hydbi",nlev), hydetai("hydetai",nlev); + ElData wrk("wrk",nlev+1), ed("ed",nlev+1); + ExecView ps("ps"); + const Real ps0 = h.ps0; + + const auto ps_m = Kokkos::create_mirror_view(ps); + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) { + ps_m(i,j) = td.urand(0.5, 1.2)*ps0; + for (int k = 0; k < nlev; ++k) { + hydai.r[k] = h.dai[k]; + hydbi.r[k] = h.dbi[k]; + hydetai.r[k] = h.detai[k]; + } + for (int k = 0; k <= nlev; ++k) + ed.r(i,j,k) = (i-j)*h.etai[k] + 0.3; + } + Kokkos::deep_copy(ps, ps_m); + hydai.h2d(); hydbi.h2d(); hydetai.h2d(); + ed.h2d(); + + const auto policy = get_test_team_policy(1, nlev); + const auto f = KOKKOS_LAMBDA(const cti::MT& team) { + KernelVariables kv(team); + calc_etadotmid_from_etadotdpdnint( + kv, nlev, ps0, hydai.d, hydbi.d, hydetai.d, ps, wrk.d, ed.d); + }; + Kokkos::parallel_for(policy, f); + Kokkos::fence(); + ed.d2h(); + + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) { + const auto den = h.a_eta*h.ps0 + h.b_eta*ps_m(i,j); + for (int k = 0; k < nlev; ++k) { + const auto ed_true = ((i-j)*h.etam[k] + 0.3)/den; + if (std::abs(ed.r(i,j,k) - ed_true) > tol*(10/den)) ++nerr; + } + } + } + + return nerr; +} + +} // namespace anon + +#define comunittest(f) do { \ + ne = f(td); \ + if (ne) printf(#f " ne %d\n", ne); \ + nerr += ne; \ + } while (0) + +int ComposeTransportImpl::run_enhanced_trajectory_unit_tests () { + int nerr = 0, ne; + TestData td(*this); + comunittest(test_find_support); + comunittest(test_linterp); + comunittest(test_eta_interp); + comunittest(test_eta_to_dp); + comunittest(test_deta_caas); + comunittest(test_limit_etam); + comunittest(test_calc_ps); + comunittest(test_calc_etadotmid_from_etadotdpdnint); + return nerr; +} + +#undef comunittest + +} // namespace Homme + +#endif diff --git a/components/homme/src/share/cxx/ComposeTransportImplGeneral.cpp b/components/homme/src/share/cxx/ComposeTransportImplGeneral.cpp index ad62ca3904f3..bbffbeb8e48c 100644 --- a/components/homme/src/share/cxx/ComposeTransportImplGeneral.cpp +++ b/components/homme/src/share/cxx/ComposeTransportImplGeneral.cpp @@ -12,7 +12,8 @@ extern "C" void sl_get_params(double* nu_q, double* hv_scaling, int* hv_q, int* hv_subcycle_q, - int* limiter_option, int* cdr_check, int* geometry_type); + int* limiter_option, int* cdr_check, int* geometry_type, + int* trajectory_nsubstep); namespace Homme { @@ -46,6 +47,8 @@ void ComposeTransportImpl::setup () { m_sphere_ops = Context::singleton().get(); set_dp_tol(); + setup_enhanced_trajectory(); + nslot = calc_nslot(m_geometry.num_elems()); } @@ -53,6 +56,11 @@ void ComposeTransportImpl::reset (const SimulationParams& params) { const auto num_elems = Context::singleton().get().get_num_local_elements(); const bool independent_time_steps = params.dt_tracer_factor > params.dt_remap_factor; + + sl_get_params(&m_data.nu_q, &m_data.hv_scaling, &m_data.hv_q, &m_data.hv_subcycle_q, + &m_data.limiter_option, &m_data.cdr_check, &m_data.geometry_type, + &m_data.trajectory_nsubstep); + if (independent_time_steps != m_data.independent_time_steps || m_data.nelemd != num_elems || m_data.qsize != params.qsize) { const auto& g = m_geometry; @@ -61,7 +69,14 @@ void ComposeTransportImpl::reset (const SimulationParams& params) { const auto& d = m_derived; const auto nel = num_elems; const auto nlev = NUM_LEV*packn; - m_data.dep_pts = DeparturePoints("dep_pts", nel); + const int ndim = (m_data.trajectory_nsubstep == 0 ? + 3 : + (independent_time_steps ? 4 : 3)); + m_data.dep_pts = DeparturePoints("dep_pts", nel, num_phys_lev, np, np, ndim); + if (m_data.trajectory_nsubstep > 0) + m_data.vnode = DeparturePoints("vnode", nel, num_phys_lev, np, np, ndim); + if (m_data.trajectory_nsubstep > 1) + m_data.vdep = DeparturePoints("vdep" , nel, num_phys_lev, np, np, ndim); homme::compose::set_views( g.m_spheremp, homme::compose::SetView (reinterpret_cast(d.m_dp.data()), @@ -76,8 +91,9 @@ void ComposeTransportImpl::reset (const SimulationParams& params) { nel, t.qdp.extent_int(1), t.qdp.extent_int(2), np, np, nlev), homme::compose::SetView (reinterpret_cast(t.Q.data()), nel, t.Q.extent_int(1), np, np, nlev), - m_data.dep_pts); + m_data.dep_pts, m_data.vnode, m_data.vdep, ndim); } + m_data.independent_time_steps = independent_time_steps; if (m_data.nelemd == num_elems && m_data.qsize == params.qsize) return; @@ -86,8 +102,6 @@ void ComposeTransportImpl::reset (const SimulationParams& params) { "SL transport requires qsize > 0; if qsize == 0, use Eulerian."); m_data.nelemd = num_elems; - sl_get_params(&m_data.nu_q, &m_data.hv_scaling, &m_data.hv_q, &m_data.hv_subcycle_q, - &m_data.limiter_option, &m_data.cdr_check, &m_data.geometry_type); Errors::runtime_check(m_data.hv_q >= 0 && m_data.hv_q <= m_data.qsize, "semi_lagrange_hv_q should be in [0, qsize]."); Errors::runtime_check(m_data.hv_subcycle_q >= 0, @@ -113,17 +127,18 @@ void ComposeTransportImpl::reset (const SimulationParams& params) { int ComposeTransportImpl::requested_buffer_size () const { // FunctorsBuffersManager wants the size in terms of sizeof(Real). - return (3*Buf1::shmem_size(nslot) + - 2*Buf2::shmem_size(nslot))/sizeof(Real); + return (m_data.n_buf1*Buf1Alloc::shmem_size(nslot) + + m_data.n_buf2*Buf2::shmem_size(nslot))/sizeof(Real); } void ComposeTransportImpl::init_buffers (const FunctorsBuffersManager& fbm) { Scalar* mem = reinterpret_cast(fbm.get_memory()); - for (int i = 0; i < 3; ++i) { - m_data.buf1[i] = Buf1(mem, nslot); - mem += Buf1::shmem_size(nslot)/sizeof(Scalar); + for (int i = 0; i < m_data.n_buf1; ++i) { + m_data.buf1o[i] = Buf1o(mem, nslot); + m_data.buf1e[i] = Buf1e(mem, nslot); // use the same memory + mem += Buf1Alloc::shmem_size(nslot)/sizeof(Scalar); } - for (int i = 0; i < 2; ++i) { + for (int i = 0; i < m_data.n_buf2; ++i) { m_data.buf2[i] = Buf2(mem, nslot); mem += Buf2::shmem_size(nslot)/sizeof(Scalar); } @@ -148,16 +163,29 @@ void ComposeTransportImpl::init_boundary_exchanges () { be->registration_completed(); } - for (int i = 0; i < 2; ++i) { - m_v_dss_be[i] = std::make_shared(); - auto be = m_v_dss_be[i]; - be->set_label(std::string("ComposeTransport-v-DSS-" + std::to_string(i))); - be->set_diagnostics_level(sp.internal_diagnostics_level); - be->set_buffers_manager(bm_exchange); - be->set_num_fields(0, 0, 2 + (i ? 1 : 0)); - be->register_field(m_derived.m_vstar, 2, 0); - if (i) be->register_field(m_derived.m_divdp); - be->registration_completed(); + if (m_data.trajectory_nsubstep == 0) { + for (int i = 0; i < 2; ++i) { + m_v_dss_be[i] = std::make_shared(); + auto be = m_v_dss_be[i]; + be->set_label(std::string("ComposeTransport-v-DSS-" + std::to_string(i))); + be->set_diagnostics_level(sp.internal_diagnostics_level); + be->set_buffers_manager(bm_exchange); + be->set_num_fields(0, 0, 2+i); + be->register_field(m_derived.m_vstar, 2, 0); + if (i) be->register_field(m_derived.m_divdp); + be->registration_completed(); + } + } else { + for (int i = 0; i < 2; ++i) { + m_v_dss_be[i] = std::make_shared(); + auto be = m_v_dss_be[i]; + be->set_label(std::string("ComposeTransport-v-DSS-" + std::to_string(i))); + be->set_diagnostics_level(sp.internal_diagnostics_level); + be->set_buffers_manager(bm_exchange); + be->set_num_fields(0, 0, 3+i); + be->register_field(m_tracers.qtens_biharmonic, 3+i, 0); + be->registration_completed(); + } } // For optional HV applied to q. @@ -181,10 +209,14 @@ void ComposeTransportImpl::init_boundary_exchanges () { void ComposeTransportImpl::run (const TimeLevel& tl, const Real dt) { GPTLstart("compose_transport"); - calc_trajectory(tl.np1, dt); + if (m_data.trajectory_nsubstep == 0) + calc_trajectory(tl.np1, dt); + else + calc_enhanced_trajectory(tl.np1, dt); GPTLstart("compose_isl"); homme::compose::advect(tl.np1, tl.n0_qdp, tl.np1_qdp); + Kokkos::fence(); GPTLstop("compose_isl"); if (m_data.hv_q > 0 && m_data.nu_q > 0) { diff --git a/components/homme/src/share/cxx/ComposeTransportImplTrajectory.cpp b/components/homme/src/share/cxx/ComposeTransportImplTrajectory.cpp index aa91c752a7c6..197ec7528a42 100644 --- a/components/homme/src/share/cxx/ComposeTransportImplTrajectory.cpp +++ b/components/homme/src/share/cxx/ComposeTransportImplTrajectory.cpp @@ -14,75 +14,19 @@ namespace Homme { using cti = ComposeTransportImpl; - -KOKKOS_FUNCTION -static void ugradv_sphere ( - const SphereOperators& sphere_ops, const KernelVariables& kv, - const typename ViewConst >::type& vec_sphere2cart, - // velocity, latlon - const typename ViewConst >::type& u, - const typename ViewConst >::type& v, - const ExecViewUnmanaged& v_cart, - const ExecViewUnmanaged& ugradv_cart, - // [u dot grad] v, latlon - const ExecViewUnmanaged& ugradv) -{ - for (int d_cart = 0; d_cart < 3; ++d_cart) { - const auto f1 = [&] (const int i, const int j, const int k) { - v_cart(i,j,k) = (vec_sphere2cart(0,d_cart,i,j) * v(0,i,j,k) + - vec_sphere2cart(1,d_cart,i,j) * v(1,i,j,k)); - }; - cti::loop_ijk(kv, f1); - kv.team_barrier(); - - sphere_ops.gradient_sphere(kv, v_cart, ugradv_cart); - - const auto f2 = [&] (const int i, const int j, const int k) { - if (d_cart == 0) ugradv(0,i,j,k) = ugradv(1,i,j,k) = 0; - for (int d_latlon = 0; d_latlon < 2; ++d_latlon) - ugradv(d_latlon,i,j,k) += - vec_sphere2cart(d_latlon,d_cart,i,j)* - (u(0,i,j,k) * ugradv_cart(0,i,j,k) + u(1,i,j,k) * ugradv_cart(1,i,j,k)); - }; - cti::loop_ijk(kv, f2); - } -} - -typedef typename ViewConst >::type CSNlev; -typedef typename ViewConst >::type CRNlev; -typedef typename ViewConst >::type CSNlevp; -typedef typename ViewConst >::type CRNlevp; -typedef typename ViewConst >::type CS2Nlev; -typedef ExecViewUnmanaged SNlev; -typedef ExecViewUnmanaged RNlev; -typedef ExecViewUnmanaged SNlevp; -typedef ExecViewUnmanaged RNlevp; -typedef ExecViewUnmanaged S2Nlev; -typedef ExecViewUnmanaged R2Nlev; -typedef ExecViewUnmanaged S2Nlevp; - -/* Form a 3rd-degree Lagrange polynomial over (x(k-1:k+1), y(k-1:k+1)) and set - yi(k) to its derivative at x(k). yps(:,:,0) is not written. - */ -KOKKOS_FUNCTION static void approx_derivative ( - const KernelVariables& kv, const CSNlevp& xs, const CSNlevp& ys, - const SNlev& yps) // yps(:,:,0) is undefined -{ - CRNlevp x(cti::cpack2real(xs)); - CRNlevp y(cti::cpack2real(ys)); - RNlev yp(cti::pack2real(yps)); - const auto f = [&] (const int i, const int j, const int k) { - if (k == 0) return; - const auto& xkm1 = x(i,j,k-1); - const auto& xk = x(i,j,k ); // also the interpolation point - const auto& xkp1 = x(i,j,k+1); - yp(i,j,k) = (y(i,j,k-1)*(( 1 /(xkm1 - xk ))*((xk - xkp1)/(xkm1 - xkp1))) + - y(i,j,k )*(( 1 /(xk - xkm1))*((xk - xkp1)/(xk - xkp1)) + - ((xk - xkm1)/(xk - xkm1))*( 1 /(xk - xkp1))) + - y(i,j,k+1)*(((xk - xkm1)/(xkp1 - xkm1))*( 1 /(xkp1 - xk )))); - }; - cti::loop_ijk(kv, f); -} +using CSNlev = cti::CSNlev; +using CRNlev = cti::CRNlev; +using CSNlevp = cti::CSNlevp; +using CRNlevp = cti::CRNlevp; +using CS2Nlev = cti::CS2Nlev; +using CR2Nlev = cti::CR2Nlev; +using SNlev = cti::SNlev; +using RNlev = cti::RNlev; +using SNlevp = cti::SNlevp; +using RNlevp = cti::RNlevp; +using S2Nlev = cti::S2Nlev; +using R2Nlev = cti::R2Nlev; +using S2Nlevp = cti::S2Nlevp; // Pad by an amount ~ smallest level to keep the computed dp > 0. void ComposeTransportImpl::set_dp_tol () { @@ -234,31 +178,11 @@ KOKKOS_FUNCTION static void calc_vertically_lagrangian_levels ( }; cti::loop_ijk(kv, f); } - + kv.team_barrier(); sphere_ops.divergence_sphere(kv, vdp, divdp); - + kv.team_barrier(); RNlevp edds(cti::pack2real(edd)), divdps(cti::pack2real(divdp)); - const auto f = [&] (const int idx) { - const int i = idx / NP, j = idx % NP; - const auto r = [&] (const int k, Real& dps, const bool final) { - assert(k != 0 || dps == 0); - if (final) edds(i,j,k) = dps; - dps += divdps(i,j,k); - }; - Dispatch<>::parallel_scan(kv.team, cti::num_phys_lev, r); - const int kend = cti::num_phys_lev - 1; - const Real dps = edds(i,j,kend) + divdps(i,j,kend); - assert(hybrid_bi(0)[0] == 0); - const auto s = [&] (const int kp) { - edd(i,j,kp) = hybrid_bi(kp)*dps - edd(i,j,kp); - if (kp == 0) edd(i,j,kp)[0] = 0; - }; - parallel_for(tvr, s); - assert(edds(i,j,0) == 0); - const int bottom = cti::num_phys_lev; - edds(i,j,bottom) = 0; - }; - parallel_for(ttr, f); + cti::calc_eta_dot_dpdn(kv, hybrid_bi, divdps, edd, edds); } // Use p0 as the reference coordinate system. p0 differs from p1 by B(eta) @@ -278,11 +202,11 @@ KOKKOS_FUNCTION static void calc_vertically_lagrangian_levels ( // Gradient of eta_dot_dpdn = p_eta deta/dt at final time w.r.t. p at initial // time. const auto& ptp0 = dprecon; - approx_derivative(kv, pref, *eta_dot_dpdn[1], ptp0); + cti::approx_derivative(kv, pref, *eta_dot_dpdn[1], ptp0); { const auto& edd = *eta_dot_dpdn[0]; - const R2Nlev vstars(cti::pack2real(vstar)); + const CR2Nlev vstars(cti::cpack2real(vstar)); const auto f_v = [&] (const int i, const int j, const int kp) { // Horizontal velocity at initial time. Scalar v[2]; @@ -353,9 +277,9 @@ void ComposeTransportImpl::calc_trajectory (const int np1, const Real dt) { const auto m_vstar = m_derived.m_vstar; const auto tu_ne = m_tu_ne; { // Calculate midpoint velocity. - const auto buf1a = m_data.buf1[0]; const auto buf1b = m_data.buf1[1]; - const auto buf1c = m_data.buf1[2]; const auto buf2a = m_data.buf2[0]; - const auto buf2b = m_data.buf2[1]; + const auto buf1a = m_data.buf1o[0]; const auto buf1b = m_data.buf1o[1]; + const auto buf1c = m_data.buf1o[2]; + const auto buf2a = m_data.buf2[0]; const auto buf2b = m_data.buf2[1]; const auto m_spheremp = geo.m_spheremp; const auto m_rspheremp = geo.m_rspheremp; const auto m_v = m_state.m_v; @@ -457,7 +381,7 @@ void ComposeTransportImpl::calc_trajectory (const int np1, const Real dt) { const auto vstar = Homme::subview(m_vstar, ie); const auto vec_sphere2cart = Homme::subview(m_vec_sph2cart, ie); const auto sphere_cart = Homme::subview(m_sphere_cart, ie); - const auto dep_pts = Homme::subview(m_dep_pts, ie); + const auto dep_pts = m_dep_pts; const auto f = [&] (const int i, const int j, const int k) { // dp = p1 - dt v/scale_factor Scalar dp[3]; @@ -476,7 +400,7 @@ void ComposeTransportImpl::calc_trajectory (const int np1, const Real dt) { // No vec call for sqrt. const auto r = is_sphere ? std::sqrt(r2[s]) : 1; for (int d = 0; d < 3; ++d) - dep_pts(oss,i,j,d) = dp[d][s]/r; + dep_pts(ie,oss,i,j,d) = dp[d][s]/r; } }; cti::loop_ijk(kv, f); @@ -511,7 +435,7 @@ static int test_approx_derivative () { { // Run approx_derivative. const auto f = KOKKOS_LAMBDA (const cti::MT& team) { KernelVariables kv(team); - approx_derivative(kv, xp, yp, yip); + cti::approx_derivative(kv, xp, yp, yip); }; Kokkos::fence(); Kokkos::parallel_for(policy, f); diff --git a/components/homme/src/share/cxx/ExecSpaceDefs.cpp b/components/homme/src/share/cxx/ExecSpaceDefs.cpp index 4f3d97135fea..dbe82d8758b7 100644 --- a/components/homme/src/share/cxx/ExecSpaceDefs.cpp +++ b/components/homme/src/share/cxx/ExecSpaceDefs.cpp @@ -14,7 +14,7 @@ #include "utilities/MathUtils.hpp" #ifdef KOKKOS_ENABLE_CUDA -# include +#include #endif #ifdef KOKKOS_ENABLE_HIP @@ -22,7 +22,7 @@ #endif #ifdef KOKKOS_ENABLE_SYCL -#include +#include #endif namespace Homme { @@ -32,56 +32,46 @@ namespace Homme { // own. As a side benefit, we'll end up running on GPU platforms optimally // without having to specify --kokkos-ndevices on the command line. void initialize_kokkos () { - // This is in fact const char*, but Kokkos::initialize requires char*. - std::vector args; - - // This is the only way to get the round-robin rank assignment Kokkos + // Count up our devices. + // This is the only way to get the round-robin rank assignment Kokkos // provides, as that algorithm is hardcoded in Kokkos::initialize(int& narg, // char* arg[]). Once the behavior is exposed in the InitArguments version of // initialize, we can remove this string code. // If for some reason we're running on a GPU platform, have Cuda enabled, // but are using a different execution space, this initialization is still // OK. The rank gets a GPU assigned and simply will ignore it. -#ifdef KOKKOS_ENABLE_CUDA - int nd; + int nd = 1; +#ifdef HOMMEXX_ENABLE_GPU +# if defined KOKKOS_ENABLE_CUDA const auto ret = cudaGetDeviceCount(&nd); - if (ret != cudaSuccess) { - // It isn't a big deal if we can't get the device count. - nd = 1; - } -#elif defined(KOKKOS_ENABLE_HIP) - int nd; + const bool ok = (ret == cudaSuccess); +# elif defined KOKKOS_ENABLE_HIP const auto ret = hipGetDeviceCount(&nd); - if (ret != hipSuccess) { + const bool ok = (ret == hipSuccess); +# elif defined KOKKOS_ENABLE_SYCL + nd = 0; + auto gpu_devs = sycl::device::get_devices(sycl::info::device_type::gpu); + for (auto &dev : gpu_devs) { + if (dev.get_info() > 0) { + auto subDevs = dev.create_sub_devices(sycl::info::partition_affinity_domain::numa); + nd += subDevs.size(); + } else { + nd++; + } + } + const bool ok = true; +# endif + if (not ok) { // It isn't a big deal if we can't get the device count. nd = 1; } -#elif defined(KOKKOS_ENABLE_SYCL) +#endif // HOMMEXX_ENABLE_GPU -//https://developer.codeplay.com/products/computecpp/ce/2.11.0/guides/sycl-for-cuda-developers/migrating-from-cuda-to-sycl - -//to make it build - int nd = 1; - -#endif - - -#ifdef HOMMEXX_ENABLE_GPU - std::stringstream ss; - ss << "--kokkos-num-devices=" << nd; - const auto key = ss.str(); - std::vector str(key.size()+1); - std::copy(key.begin(), key.end(), str.begin()); - str.back() = 0; - args.push_back(const_cast(str.data())); -#endif - - - const char* silence = "--kokkos-disable-warnings"; - args.push_back(const_cast(silence)); - - int narg = args.size(); - Kokkos::initialize(narg, args.data()); + auto const settings = Kokkos::InitializationSettings() + .set_map_device_id_by("mpi_rank") + .set_num_devices(nd) + .set_disable_warnings(true); + Kokkos::initialize(settings); } ThreadPreferences::ThreadPreferences () @@ -214,7 +204,7 @@ team_num_threads_vectors (const int num_parallel_iterations, #endif min_num_warps = std::min(min_num_warps, max_num_warps); - + return Parallel::team_num_threads_vectors_for_gpu( num_warps_device, num_threads_warp, min_num_warps, max_num_warps, diff --git a/components/homme/src/share/cxx/GllFvRemapImpl.hpp b/components/homme/src/share/cxx/GllFvRemapImpl.hpp index 11738b2bf455..38899fcf499d 100644 --- a/components/homme/src/share/cxx/GllFvRemapImpl.hpp +++ b/components/homme/src/share/cxx/GllFvRemapImpl.hpp @@ -39,7 +39,6 @@ struct GllFvRemapImpl { enum : int { num_lev_aligned = num_lev_pack*packn }; enum : int { num_levp_aligned = max_num_lev_pack*packn }; enum : int { num_phys_lev = NUM_PHYSICAL_LEV }; - enum : int { num_work = 12 }; typedef GllFvRemap::Phys1T Phys1T; typedef GllFvRemap::Phys2T Phys2T; diff --git a/components/homme/src/share/cxx/HommexxEnums.hpp b/components/homme/src/share/cxx/HommexxEnums.hpp index 06abbf35adbc..f41ed9d3651e 100644 --- a/components/homme/src/share/cxx/HommexxEnums.hpp +++ b/components/homme/src/share/cxx/HommexxEnums.hpp @@ -40,6 +40,12 @@ enum class ComparisonOp { // =================== Run parameters enums ====================== // +//todo-repo-unification Remove this enum in favor of bool +// SimulationParams::use_moisture once we change EAMxx to use +// use_moisture. Search "todo-repo-unification" for other bits of code to +// remove. +enum class MoistDry { MOIST, DRY }; + enum class ForcingAlg : int { FORCING_OFF =-1, FORCING_0 = 0, diff --git a/components/homme/src/share/cxx/Tracers.cpp b/components/homme/src/share/cxx/Tracers.cpp index cac9b3b15d7e..7d338229d519 100644 --- a/components/homme/src/share/cxx/Tracers.cpp +++ b/components/homme/src/share/cxx/Tracers.cpp @@ -32,7 +32,10 @@ void Tracers::init(const int num_elems, const int num_tracers) nt = num_tracers; qdp = decltype(qdp)("tracers mass", num_elems); - qtens_biharmonic = decltype(qtens_biharmonic)("qtens(_biharmonic)", num_elems); + // Also used in ComposeTransportImplEnhancedTrajectory for communication, + // where 4 slots are needed. + qtens_biharmonic = decltype(qtens_biharmonic)( + "qtens(_biharmonic)", num_elems, std::max(4, num_tracers)); qlim = decltype(qlim)("qlim", num_elems); Q = decltype(Q)("tracers concentration", num_elems,num_tracers); diff --git a/components/homme/src/share/cxx/Tracers.hpp b/components/homme/src/share/cxx/Tracers.hpp index 78f93c06a703..592350894ab8 100644 --- a/components/homme/src/share/cxx/Tracers.hpp +++ b/components/homme/src/share/cxx/Tracers.hpp @@ -35,8 +35,8 @@ struct Tracers { bool inited () const { return m_inited; } ExecViewManaged qdp; - ExecViewManaged qtens_biharmonic; // Also doubles as just qtens. - ExecViewManaged qlim; + ExecViewManaged qtens_biharmonic; // Also doubles as just qtens. + ExecViewManaged qlim; ExecViewManaged Q; ExecViewManaged fq; diff --git a/components/homme/src/share/cxx/prim_step.cpp b/components/homme/src/share/cxx/prim_step.cpp index 9e6afdec0d1a..c5c75e54a354 100644 --- a/components/homme/src/share/cxx/prim_step.cpp +++ b/components/homme/src/share/cxx/prim_step.cpp @@ -183,7 +183,9 @@ void prim_step_flexible (const Real dt, const bool compute_diagnostics) { Errors::err_not_implemented); } else { // Remap dynamics variables but not tracers. + GPTLstart("tl-sc vertical_remap"); vertical_remap(dt_remap); + GPTLstop("tl-sc vertical_remap"); } } } diff --git a/components/homme/src/share/cxx/utilities/SubviewUtils.hpp b/components/homme/src/share/cxx/utilities/SubviewUtils.hpp index 76171a1432c3..c78517a3b82d 100644 --- a/components/homme/src/share/cxx/utilities/SubviewUtils.hpp +++ b/components/homme/src/share/cxx/utilities/SubviewUtils.hpp @@ -352,6 +352,21 @@ subview(ViewType +KOKKOS_INLINE_FUNCTION ViewUnmanaged +subview(ViewType v_in, + int ie, int idim0) { + assert(v_in.data() != nullptr); + assert(ie < v_in.extent_int(0)); + assert(ie >= 0); + assert(idim0 < v_in.extent_int(1)); + assert(idim0 >= 0); + return ViewUnmanaged( + &v_in.impl_map().reference(ie, idim0, 0, 0, 0, 0)); +} + // Force a subview to be const template KOKKOS_INLINE_FUNCTION diff --git a/components/homme/src/share/cxx/utilities/scream_tridiag.hpp b/components/homme/src/share/cxx/utilities/scream_tridiag.hpp index 26221db39552..f302432ef790 100644 --- a/components/homme/src/share/cxx/utilities/scream_tridiag.hpp +++ b/components/homme/src/share/cxx/utilities/scream_tridiag.hpp @@ -120,7 +120,7 @@ int get_team_nthr (const TeamMember& team) { return team.team_size(); } -// Impl details for Nvidia and AMD GPUs. +// Impl details for Nvidia, AMD and Intel GPUs. template KOKKOS_FORCEINLINE_FUNCTION int get_thread_id_within_team_gpu (const TeamMember& team) { diff --git a/components/homme/src/share/cxx/vector/vector_pragmas.hpp b/components/homme/src/share/cxx/vector/vector_pragmas.hpp index 3a0ae7f97ee6..fc58b819a293 100644 --- a/components/homme/src/share/cxx/vector/vector_pragmas.hpp +++ b/components/homme/src/share/cxx/vector/vector_pragmas.hpp @@ -7,11 +7,18 @@ #ifndef HOMMEXX_VECTOR_PRAGMAS_HPP #define HOMMEXX_VECTOR_PRAGMAS_HPP -#if defined(__INTEL_COMPILER) +#include "Config.hpp" + +#if defined(__INTEL_COMPILER) || defined(__INTEL_CLANG_COMPILER) || defined(__INTEL_LLVM_COMPILER) #define VECTOR_IVDEP_LOOP _Pragma("ivdep") #define ALWAYS_VECTORIZE_LOOP _Pragma("vector always") #define VECTOR_SIMD_LOOP _Pragma("omp simd") +#if HOMMEXX_VECTOR_SIZE == 1 +# define VECTOR_SIMD_LOOP +#else +# define VECTOR_SIMD_LOOP _Pragma("omp simd") +#endif #elif defined(__GNUG__) && !defined(__NVCC__) #if(__GNUG__ == 4 && __GNUC_MINOR__ >= 9) || __GNUG__ > 4 diff --git a/components/homme/src/share/namelist_mod.F90 b/components/homme/src/share/namelist_mod.F90 index 1d47090182ba..8f3eff70e3e1 100644 --- a/components/homme/src/share/namelist_mod.F90 +++ b/components/homme/src/share/namelist_mod.F90 @@ -46,6 +46,10 @@ module namelist_mod semi_lagrange_cdr_check, & semi_lagrange_hv_q, & semi_lagrange_nearest_point_lev, & + semi_lagrange_halo, & + semi_lagrange_trajectory_nsubstep, & + semi_lagrange_trajectory_nvelocity, & + semi_lagrange_diagnostics, & tstep_type, & cubed_sphere_map, & qsplit, & @@ -272,6 +276,11 @@ subroutine readnl(par) semi_lagrange_cdr_check, & semi_lagrange_hv_q, & semi_lagrange_nearest_point_lev, & + semi_lagrange_halo, & + semi_lagrange_trajectory_nsubstep, & + semi_lagrange_trajectory_nvelocity, & + semi_lagrange_diagnostics, & + semi_lagrange_hv_q, & tstep_type, & cubed_sphere_map, & qsplit, & @@ -426,6 +435,13 @@ subroutine readnl(par) se_ftype = ftype ! MNL: For non-CAM runs, ftype=0 in control_mod nsplit = 1 pertlim = 0.0_real_kind +#else + se_partmethod = SFCURVE + se_ne = 0 + se_ne_x = 0 + se_ne_y = 0 + se_lx = 0 + se_ly = 0 #endif sub_case = 1 numnodes = -1 @@ -446,6 +462,10 @@ subroutine readnl(par) semi_lagrange_cdr_check = .false. semi_lagrange_hv_q = 1 semi_lagrange_nearest_point_lev = 256 + semi_lagrange_halo = 2 + semi_lagrange_trajectory_nsubstep = 0 + semi_lagrange_trajectory_nvelocity = -1 + semi_lagrange_diagnostics = 0 disable_diagnostics = .false. se_fv_phys_remap_alg = 1 internal_diagnostics_level = 0 @@ -856,6 +876,10 @@ subroutine readnl(par) call MPI_bcast(semi_lagrange_cdr_check ,1,MPIlogical_t,par%root,par%comm,ierr) call MPI_bcast(semi_lagrange_hv_q ,1,MPIinteger_t,par%root,par%comm,ierr) call MPI_bcast(semi_lagrange_nearest_point_lev ,1,MPIinteger_t,par%root,par%comm,ierr) + call MPI_bcast(semi_lagrange_halo ,1,MPIinteger_t,par%root,par%comm,ierr) + call MPI_bcast(semi_lagrange_trajectory_nsubstep ,1,MPIinteger_t,par%root,par%comm,ierr) + call MPI_bcast(semi_lagrange_trajectory_nvelocity ,1,MPIinteger_t,par%root,par%comm,ierr) + call MPI_bcast(semi_lagrange_diagnostics ,1,MPIinteger_t,par%root,par%comm,ierr) call MPI_bcast(tstep_type,1,MPIinteger_t ,par%root,par%comm,ierr) call MPI_bcast(cubed_sphere_map,1,MPIinteger_t ,par%root,par%comm,ierr) call MPI_bcast(qsplit,1,MPIinteger_t ,par%root,par%comm,ierr) @@ -1172,6 +1196,10 @@ subroutine readnl(par) write(iulog,*)"readnl: semi_lagrange_cdr_check = ",semi_lagrange_cdr_check write(iulog,*)"readnl: semi_lagrange_hv_q = ",semi_lagrange_hv_q write(iulog,*)"readnl: semi_lagrange_nearest_point_lev = ",semi_lagrange_nearest_point_lev + write(iulog,*)"readnl: semi_lagrange_halo = ",semi_lagrange_halo + write(iulog,*)"readnl: semi_lagrange_trajectory_nsubstep = ",semi_lagrange_trajectory_nsubstep + write(iulog,*)"readnl: semi_lagrange_trajectory_nvelocity = ",semi_lagrange_trajectory_nvelocity + write(iulog,*)"readnl: semi_lagrange_diagnostics = ",semi_lagrange_diagnostics write(iulog,*)"readnl: tstep_type = ",tstep_type write(iulog,*)"readnl: theta_advect_form = ",theta_advect_form write(iulog,*)"readnl: vtheta_thresh = ",vtheta_thresh diff --git a/components/homme/src/share/parallel_mod.F90 b/components/homme/src/share/parallel_mod.F90 index 69016a8a0636..49ff71779a81 100644 --- a/components/homme/src/share/parallel_mod.F90 +++ b/components/homme/src/share/parallel_mod.F90 @@ -274,6 +274,7 @@ subroutine abortmp(string) #endif #endif character*(*) string + call flush(iulog) #ifdef CAM call endrun(string) #else diff --git a/components/homme/src/share/prim_driver_base.F90 b/components/homme/src/share/prim_driver_base.F90 index df83e1cc386b..9e0363473204 100644 --- a/components/homme/src/share/prim_driver_base.F90 +++ b/components/homme/src/share/prim_driver_base.F90 @@ -46,6 +46,8 @@ module prim_driver_base public :: applyCAMforcing_tracers + public :: set_tracer_transport_derived_values + ! Service variables used to partition the mesh. ! Note: GridEdge and MeshVertex are public, cause kokkos targets need to access them type (GridVertex_t), pointer :: GridVertex(:) @@ -1316,7 +1318,7 @@ subroutine prim_step_flexible(hybrid, elem, nets, nete, dt, tl, hvcoord, compute use hybvcoord_mod, only: hvcoord_t use parallel_mod, only: abortmp use prim_advance_mod, only: prim_advance_exp, applycamforcing_dynamics - use prim_advection_mod, only: prim_advec_tracers_remap + use prim_advection_mod, only: prim_advec_tracers_observe_velocity, prim_advec_tracers_remap use reduction_mod, only: parallelmax use time_mod, only: TimeLevel_t, timelevel_update, timelevel_qdp use prim_state_mod, only: prim_printstate @@ -1334,7 +1336,7 @@ subroutine prim_step_flexible(hybrid, elem, nets, nete, dt, tl, hvcoord, compute real(kind=real_kind) :: dt_q, dt_remap, dp(np,np,nlev) integer :: ie, q, k, n, n0_qdp, np1_qdp - logical :: compute_diagnostics_it, apply_forcing + logical :: compute_diagnostics_it, apply_forcing, observe dt_q = dt*dt_tracer_factor if (dt_remap_factor == 0) then @@ -1392,11 +1394,13 @@ subroutine prim_step_flexible(hybrid, elem, nets, nete, dt, tl, hvcoord, compute call prim_advance_exp(elem, deriv1, hvcoord, hybrid, dt, tl, nets, nete, & compute_diagnostics_it) + observe = .false. if (dt_remap_factor == 0) then ! Set np1_qdp to -1. Since dt_remap == 0, the only part of ! vertical_remap that is active is the updates to ! ps_v(:,:,np1) and dp3d(:,:,:,np1). call vertical_remap(hybrid, elem, hvcoord, dt_remap, tl%np1, -1, nets, nete) + observe = .true. else if (modulo(n, dt_remap_factor) == 0) then if (compute_diagnostics) call run_diagnostics(elem,hvcoord,tl,4,.false.,nets,nete) @@ -1417,8 +1421,10 @@ subroutine prim_step_flexible(hybrid, elem, nets, nete, dt, tl, hvcoord, compute ! not tracers. call vertical_remap(hybrid, elem, hvcoord, dt_remap, tl%np1, -1, nets, nete) end if + observe = .true. end if end if + if (observe) call Prim_Advec_Tracers_observe_velocity(elem, tl, n, nets, nete) ! defer final timelevel update until after Q update. enddo call t_stopf("prim_step_dyn") diff --git a/components/homme/src/share/reduction_mod.F90 b/components/homme/src/share/reduction_mod.F90 index 83232478ef3d..72e0e89231f2 100644 --- a/components/homme/src/share/reduction_mod.F90 +++ b/components/homme/src/share/reduction_mod.F90 @@ -25,7 +25,7 @@ module reduction_mod integer :: ctr end type ReductionBuffer_ordered_1d_t - public :: ParallelMin,ParallelMax + public :: ParallelMin,ParallelMax,ParallelSum !type (ReductionBuffer_ordered_1d_t), public :: red_sum type (ReductionBuffer_int_1d_t), public :: red_max_int @@ -51,6 +51,9 @@ module reduction_mod module procedure ParallelMax0d module procedure ParallelMax0d_int end interface + interface ParallelSum + module procedure ParallelSum0d_int + end interface interface pmax_mt module procedure pmax_mt_int_1d @@ -61,6 +64,10 @@ module reduction_mod module procedure pmin_mt_r_1d end interface + interface psum_mt + module procedure psum_mt_int_1d + end interface + interface InitReductionBuffer module procedure InitReductionBuffer_int_1d module procedure InitReductionBuffer_r_1d @@ -179,7 +186,19 @@ function ParallelMax0d_int(data,hybrid) result(pmax) end function ParallelMax0d_int + !**************************************************************** + function ParallelSum0d_int(data,hybrid) result(psum) + use hybrid_mod, only : hybrid_t + implicit none + integer , intent(in) :: data + type (hybrid_t), intent(in) :: hybrid + integer :: psum + integer :: tmp(1) + tmp(1)=data + call psum_mt(red_sum_int,tmp,1,hybrid) + psum = red_sum_int%buf(1) + end function ParallelSum0d_int !**************************************************************** subroutine InitReductionBuffer_int_1d(red,len) @@ -486,7 +505,53 @@ subroutine pmin_mt_r_1d(red,redp,len,hybrid) !$OMP BARRIER end subroutine pmin_mt_r_1d + ! ======================================= + ! psum_mt: + ! + ! thread safe, parallel reduce sum of a + ! one dimensional INTEGER reduction vector + ! ======================================= + subroutine psum_mt_int_1d(red,redp,len,hybrid) + use hybrid_mod, only : hybrid_t +#ifdef _MPI + use parallel_mod, only: mpi_sum, mpiinteger_t +#endif + use parallel_mod, only: abortmp + + type (ReductionBuffer_int_1d_t) :: red ! shared memory reduction buffer struct + integer, intent(in) :: len ! buffer length + integer, intent(inout) :: redp(len) ! thread private vector of partial sum + type (hybrid_t), intent(in) :: hybrid ! parallel handle + + ! Local variables + integer ierr, k + + if (len>red%len) call abortmp('ERROR: threadsafe reduction buffer too small') + + !$OMP BARRIER + ! the first and fastest thread performs initializing copy + !$OMP SINGLE + red%buf(1:len) = redp(1:len) + red%ctr = hybrid%ithr + !$OMP END SINGLE + !$OMP CRITICAL (CRITMAXINT) + if (hybrid%ithr /= red%ctr) then + do k=1,len + red%buf(k) = red%buf(k) + redp(k) + enddo + end if + !$OMP END CRITICAL (CRITMAXINT) +#ifdef _MPI + !$OMP BARRIER + if (hybrid%ithr==0) then + call MPI_Allreduce(red%buf(1),redp,len,MPIinteger_t, & + MPI_SUM,hybrid%par%comm,ierr) + red%buf(1:len)=redp(1:len) + end if +#endif + !$OMP BARRIER + end subroutine psum_mt_int_1d ! ======================================= subroutine ElementSum_1d(res,variable,type,hybrid) diff --git a/components/homme/src/share/sl_advection.F90 b/components/homme/src/share/sl_advection.F90 index 3f6544953b66..63a476d0325e 100644 --- a/components/homme/src/share/sl_advection.F90 +++ b/components/homme/src/share/sl_advection.F90 @@ -5,12 +5,12 @@ module sl_advection use kinds, only : real_kind, int_kind use dimensions_mod, only : nlev, nlevp, np, qsize, qsize_d - use derivative_mod, only : derivative_t, gradient_sphere, divergence_sphere + use derivative_mod, only : derivative_t, gradient_sphere, divergence_sphere, ugradv_sphere use element_mod, only : element_t use hybvcoord_mod, only : hvcoord_t use time_mod, only : TimeLevel_t, TimeLevel_Qdp - use control_mod, only : integration, test_case, hypervis_order, transport_alg, limiter_option,& - vert_remap_q_alg + use control_mod, only : integration, test_case, hypervis_order, transport_alg, & + & limiter_option, vert_remap_q_alg, semi_lagrange_diagnostics use edge_mod, only : edgevpack_nlyr, edgevunpack_nlyr, edge_g use edgetype_mod, only : EdgeDescriptor_t, EdgeBuffer_t use hybrid_mod, only : hybrid_t @@ -26,18 +26,20 @@ module sl_advection private + ! Constants real(real_kind), parameter :: zero = 0.0_real_kind, fourth = 0.25_real_kind, & half = 0.5_real_kind, one = 1.0_real_kind, two = 2.0_real_kind, & eps = epsilon(1.0_real_kind) - type (cartesian3D_t), allocatable :: dep_points_all(:,:,:,:) ! (np,np,nlev,nelemd) - real(kind=real_kind), dimension(:,:,:,:,:), allocatable :: minq, maxq ! (np,np,nlev,qsize,nelemd) - logical :: is_sphere + ! Configuration. + logical :: is_sphere, enhanced_trajectory + integer :: dep_points_ndim - ! For use in make_positive. - real(kind=real_kind) :: dp_tol + ! For use in make_positive. Set at initialization to a function of hvcoord%dp0. + real(kind=real_kind) :: dp_tol, deta_tol - public :: prim_advec_tracers_remap_ALE, sl_init1, sl_vertically_remap_tracers, sl_unittest + public :: prim_advec_tracers_observe_velocity_ALE, prim_advec_tracers_remap_ALE, & + & sl_init1, sl_vertically_remap_tracers, sl_unittest ! For testing public :: calc_trajectory, dep_points_all, sphere2cart @@ -45,8 +47,36 @@ module sl_advection ! For C++ public :: sl_get_params + ! Barrier for performance analysis. Should be false in production runs. logical, parameter :: barrier = .false. + ! Bounds for shape preservation. + real(kind=real_kind), dimension(:,:,:,:,:), allocatable :: minq, maxq ! (np,np,nlev,qsize,nelemd) + + ! Trajectory velocity data. + real(kind=real_kind), dimension(:,:,:,:,:), allocatable :: vnode, vdep ! (ndim,np,np,nlev,nelemd) + real(kind=real_kind), allocatable :: dep_points_all(:,:,:,:,:) ! (ndim,np,np,nlev,nelemd) + + type :: velocity_record_t + integer :: nvel + ! Times to which velocity slots correspond, in reference time [0,dtf]. + real(kind=real_kind), allocatable :: t_vel(:) ! 1:nvel + ! For n = 1:dtf, obs_slots(n,:) = [slot1, slot2], -1 if unused. These are + ! the slots to which velocity sample n contributes. obs_slots(dtf,:) is + ! always -1. + integer, allocatable :: obs_slots(:,:) + ! obs_wts(n,:) = [wt1, wt2], 0 if unused. + real(kind=real_kind), allocatable :: obs_wts(:,:) + ! Substep end point n in 0:nsub uses velocity slots run_step(n), + ! run_step(n)-1. + integer, allocatable :: run_step(:) + ! Store state%v and state%dp3d at t_vel points. + real(kind=real_kind), allocatable :: vel(:,:,:,:,:,:) ! (np,np,2,nlev,nss,nelemd) + real(kind=real_kind), allocatable :: dp (:,:, :,:,:) ! (np,np, nlev,nss,nelemd) + end type velocity_record_t + + type(velocity_record_t) :: vrec + contains !=================================================================================================! @@ -87,7 +117,9 @@ end subroutine sphere2cart subroutine sl_init1(par, elem) use interpolate_mod, only : interpolate_tracers_init use control_mod, only : transport_alg, semi_lagrange_cdr_alg, cubed_sphere_map, & - nu_q, semi_lagrange_hv_q, semi_lagrange_cdr_check, geometry + nu_q, semi_lagrange_hv_q, semi_lagrange_cdr_check, semi_lagrange_trajectory_nsubstep, & + semi_lagrange_trajectory_nvelocity, geometry, dt_remap_factor, dt_tracer_factor, & + semi_lagrange_halo use element_state, only : timelevels use coordinate_systems_mod, only : cartesian3D_t use perf_mod, only: t_startf, t_stopf @@ -102,14 +134,13 @@ subroutine sl_init1(par, elem) #ifdef HOMME_ENABLE_COMPOSE call t_startf('sl_init1') if (transport_alg > 0) then - call sl_parse_transport_alg(transport_alg, slmm, cisl, qos, sl_test, independent_time_steps) - if (par%masterproc .and. nu_q > 0 .and. semi_lagrange_hv_q > 0) & - write(iulog,*) 'COMPOSE> use HV; nu_q, all:', nu_q, semi_lagrange_hv_q + call sl_parse_transport_alg(transport_alg, slmm, cisl, qos, sl_test, & + independent_time_steps) is_sphere = trim(geometry) /= 'plane' + enhanced_trajectory = semi_lagrange_trajectory_nsubstep > 0 + dep_points_ndim = 3 + if (enhanced_trajectory .and. independent_time_steps) dep_points_ndim = 4 nslots = nlev*qsize - ! Technically a memory leak, but the array persists for the entire - ! run, so not a big deal for now. - allocate(dep_points_all(np,np,nlev,size(elem))) do ie = 1, size(elem) ! Provide a point inside the target element. call sphere2cart(elem(ie)%spherep(2,2), pinside) @@ -130,21 +161,45 @@ subroutine sl_init1(par, elem) need_conservation = 1 call cedr_sl_init(np, nlev, qsize, qsize_d, timelevels, need_conservation) end if - allocate(minq(np,np,nlev,qsize,size(elem)), maxq(np,np,nlev,qsize,size(elem))) + allocate(minq(np,np,nlev,qsize,size(elem)), maxq(np,np,nlev,qsize,size(elem)), & + & dep_points_all(dep_points_ndim,np,np,nlev,size(elem))) + if (enhanced_trajectory) then + allocate(vnode(dep_points_ndim,np,np,nlev,size(elem)), & + & vdep (dep_points_ndim,np,np,nlev,size(elem))) + end if + call init_velocity_record(size(elem), dt_tracer_factor, dt_remap_factor, & + semi_lagrange_trajectory_nsubstep, semi_lagrange_trajectory_nvelocity, & + vrec, i) dp_tol = -one + deta_tol = -one + if (par%masterproc) then + if (nu_q > 0 .and. semi_lagrange_hv_q > 0) then + write(iulog,'(a,es13.4,i3)') 'COMPOSE> use HV; nu_q, hv_q:', & + nu_q, semi_lagrange_hv_q + end if + if (enhanced_trajectory) then + write(iulog,'(a,i3,i3,i3)') & + 'COMPOSE> dt_tracer_factor, dt_remap_factor, halo:', & + dt_tracer_factor, dt_remap_factor, semi_lagrange_halo + write(iulog,'(a,i3,i3)') & + 'COMPOSE> use enhanced trajectory; nsub, nvel:', & + semi_lagrange_trajectory_nsubstep, vrec%nvel + end if + end if endif call t_stopf('sl_init1') #endif end subroutine sl_init1 subroutine sl_get_params(nu_q_out, hv_scaling, hv_q, hv_subcycle_q, limiter_option_out, & - cdr_check, geometry_type) bind(c) + cdr_check, geometry_type, trajectory_nsubstep) bind(c) use control_mod, only: semi_lagrange_hv_q, hypervis_subcycle_q, semi_lagrange_cdr_check, & - nu_q, hypervis_scaling, limiter_option, geometry + nu_q, hypervis_scaling, limiter_option, geometry, semi_lagrange_trajectory_nsubstep use iso_c_binding, only: c_int, c_double real(c_double), intent(out) :: nu_q_out, hv_scaling - integer(c_int), intent(out) :: hv_q, hv_subcycle_q, limiter_option_out, cdr_check, geometry_type + integer(c_int), intent(out) :: hv_q, hv_subcycle_q, limiter_option_out, cdr_check, & + geometry_type, trajectory_nsubstep nu_q_out = nu_q hv_scaling = hypervis_scaling @@ -155,21 +210,143 @@ subroutine sl_get_params(nu_q_out, hv_scaling, hv_q, hv_subcycle_q, limiter_opti if (semi_lagrange_cdr_check) cdr_check = 1 geometry_type = 0 ! sphere if (trim(geometry) == "plane") geometry_type = 1 - + trajectory_nsubstep = semi_lagrange_trajectory_nsubstep end subroutine sl_get_params + subroutine init_velocity_record(nelemd, dtf, drf_param, nsub, nvel_param, v, error) + integer, intent(in) :: nelemd, dtf, drf_param, nsub, nvel_param + type (velocity_record_t), intent(out) :: v + integer, intent(out) :: error + + real(kind=real_kind) :: t_avail(0:dtf), time + integer :: nvel, drf, navail, n, i, iav + + error = 0 + drf = drf_param + if (drf == 0) drf = 1 ! drf = 0 if vertically Eulerian + nvel = nvel_param + if (nvel == -1) nvel = 2 + ((nsub-1) / 2) + nvel = min(nvel, nsub+1) + navail = dtf/drf + 1 + nvel = min(nvel, navail) + + ! nsub <= 1: No substepping. + ! nvel <= 2: Save velocity only at endpoints, as always occurs. + if (nsub <= 1 .or. nvel <= 2) then + v%nvel = 2 + return + end if + + v%nvel = nvel + allocate(v%t_vel(nvel), v%obs_slots(dtf,2), v%obs_wts(dtf,2), v%run_step(0:nsub), & + & v%vel(np,np,2,nlev,2:nvel-1,nelemd), v%dp(np,np,nlev,2:nvel-1,nelemd)) + + ! Times at which velocity data are available. + t_avail(0) = 0 + i = 1 + do n = 1, dtf + if (modulo(n, drf) == 0) then + t_avail(i) = n + i = i + 1 + end if + end do + if (i /= navail) error = 1 + + ! Times to which we associate velocity data. + do n = 1, nvel + if (modulo((n-1)*dtf, nvel-1) == 0) then + ! Cast integer values at end of calculation. + v%t_vel(n) = ((n-1)*dtf)/(nvel-1) + else + v%t_vel(n) = real((n-1)*dtf, real_kind)/(nvel-1) + end if + end do + + ! Build the tables mapping n in 1:dtf to velocity slots to accumulate into. + do n = 1, dtf-1 + v%obs_slots(n,:) = -1 + v%obs_wts(n,:) = 0 + if (modulo(n, drf) /= 0) cycle + time = n + do i = 1, navail-1 + if (time == t_avail(i)) exit + end do + iav = i + if (iav > navail-1) error = 2 + do i = 2, nvel-1 + if (t_avail(iav-1) < v%t_vel(i) .and. time > v%t_vel(i)) then + v%obs_slots(n,1) = i + v%obs_wts(n,1) = (v%t_vel(i) - t_avail(iav-1))/(t_avail(iav) - t_avail(iav-1)) + end if + if (time <= v%t_vel(i) .and. t_avail(iav+1) > v%t_vel(i)) then + v%obs_slots(n,2) = i + v%obs_wts(n,2) = (t_avail(iav+1) - v%t_vel(i))/(t_avail(iav+1) - t_avail(iav)) + end if + end do + end do + v%obs_slots(dtf,:) = -1 + v%obs_wts(dtf,:) = 0 + + ! Build table mapping n to interval to use. The trajectories go backward in + ! time, and this table reflects that. + v%run_step(0) = nvel + v%run_step(nsub) = 2 + do n = 1, nsub-1 + time = real((nsub-n)*dtf, real_kind)/nsub + do i = 1, nvel-1 + if (v%t_vel(i) <= time .and. time <= v%t_vel(i+1)) exit + end do + if (i > nvel-1) error = 4 + v%run_step(n) = i+1 + end do + end subroutine init_velocity_record + + subroutine prim_advec_tracers_observe_velocity_ALE(elem, tl, n, nets, nete) + use control_mod, only: dt_remap_factor + + type (element_t) , intent(inout) :: elem(:) + type (TimeLevel_t) , intent(in ) :: tl + integer , intent(in ) :: n ! step in 1:dt_tracer_factor + integer , intent(in ) :: nets + integer , intent(in ) :: nete + + integer :: nstore, islot, slot, k, ie + + if (vrec%nvel == 2) return + + if (n == dt_remap_factor .or. (dt_remap_factor == 0 .and. n == 1)) then + ! First observation of the tracer time step: zero accumulated quantities. + do ie = nets, nete + do slot = 2, vrec%nvel-1 + vrec%vel(:,:,:,:,slot,ie) = 0 + vrec%dp (:,:, :,slot,ie) = 0 + end do + end do + end if + + do islot = 1, 2 + slot = vrec%obs_slots(n,islot) + if (slot == -1) cycle + do ie = nets, nete + do k = 1, nlev + vrec%vel(:,:,:,k,slot,ie) = vrec%vel(:,:,:,k,slot,ie) + & + vrec%obs_wts(n,islot) * elem(ie)%state%v(:,:,:,k,tl%np1) + vrec%dp (:,:, k,slot,ie) = vrec%dp (:,:, k,slot,ie) + & + vrec%obs_wts(n,islot) * elem(ie)%state%dp3d(:,:,k,tl%np1) + end do + end do + end do + end subroutine prim_advec_tracers_observe_velocity_ALE + subroutine prim_advec_tracers_remap_ALE(elem, deriv, hvcoord, hybrid, dt, tl, nets, nete) use coordinate_systems_mod, only : cartesian3D_t, cartesian2D_t use dimensions_mod, only : max_neigh_edges use interpolate_mod, only : interpolate_tracers, minmax_tracers use control_mod, only : dt_tracer_factor, nu_q, transport_alg, semi_lagrange_hv_q, & - semi_lagrange_cdr_alg, semi_lagrange_cdr_check + semi_lagrange_cdr_alg, semi_lagrange_cdr_check, semi_lagrange_trajectory_nsubstep ! For DCMIP16 supercell test case. use control_mod, only : dcmip16_mu_q use prim_advection_base, only : advance_physical_vis -#ifdef HOMME_ENABLE_COMPOSE - use compose_mod, only : compose_h2d, compose_d2h -#endif use iso_c_binding, only : c_bool implicit none @@ -182,8 +359,6 @@ subroutine prim_advec_tracers_remap_ALE(elem, deriv, hvcoord, hybrid, dt, tl, ne integer , intent(in ) :: nets integer , intent(in ) :: nete - type(cartesian3D_t) :: dep_points (np,np) - integer :: i,j,k,l,n,q,ie,n0_qdp,np1_qdp integer :: scalar_q_bounds, info logical :: slmm, cisl, qos, sl_test, independent_time_steps @@ -194,19 +369,18 @@ subroutine prim_advec_tracers_remap_ALE(elem, deriv, hvcoord, hybrid, dt, tl, ne call t_startf('Prim_Advec_Tracers_remap_ALE') call sl_parse_transport_alg(transport_alg, slmm, cisl, qos, sl_test, independent_time_steps) - ! Until I get the DSS onto GPU, always need to h<->d. - !h2d = hybrid%par%nprocs > 1 .or. semi_lagrange_cdr_check .or. & (semi_lagrange_hv_q > 0 .and. nu_q > 0) h2d = .true. -#ifdef HOMME_ENABLE_COMPOSE d2h = compose_d2h .or. h2d h2d = compose_h2d .or. h2d -#else - d2h = h2d -#endif call TimeLevel_Qdp(tl, dt_tracer_factor, n0_qdp, np1_qdp) - call calc_trajectory(elem, deriv, hvcoord, hybrid, dt, tl, & - independent_time_steps, nets, nete) + if (enhanced_trajectory) then + call calc_enhanced_trajectory(elem, deriv, hvcoord, hybrid, dt, tl, nets, nete, & + semi_lagrange_trajectory_nsubstep, independent_time_steps) + else + call calc_trajectory(elem, deriv, hvcoord, hybrid, dt, tl, & + independent_time_steps, nets, nete) + end if call t_startf('SLMM_csl') !todo Here and in the set-pointer loop for CEDR, do just in the first call. @@ -218,14 +392,14 @@ subroutine prim_advec_tracers_remap_ALE(elem, deriv, hvcoord, hybrid, dt, tl, ne h2d, d2h) end do ! edge_g buffers are shared by SLMM, CEDR, other places in HOMME, and - ! dp_coupling in EAM. Thus, we must take care to protected threaded - ! access. In the following, "No barrier needed" comments justify why a - ! barrier isn't needed. + ! dp_coupling in EAM. Thus, we must take care to protect threaded access. In + ! the following, "No barrier needed" comments justify why a barrier isn't + ! needed. ! No barrier needed: ale_rkdss has a horiz thread barrier at the end. - call slmm_csl(nets, nete, dep_points_all, minq, maxq, info) + call slmm_csl(nets, nete, dep_points_all, dep_points_ndim, minq, maxq, info) ! No barrier needed: slmm_csl has a horiz thread barrier at the end. if (info /= 0) then - call write_velocity_data(elem, nets, nete, hybrid, deriv, dt, tl) + call write_velocity_data(elem, nets, nete, hybrid, dt, tl) call abortmp('slmm_csl returned -1; see output above for more information.') end if if (barrier) call perf_barrier(hybrid) @@ -261,7 +435,7 @@ subroutine prim_advec_tracers_remap_ALE(elem, deriv, hvcoord, hybrid, dt, tl, ne call t_stopf('CEDR') call t_startf('CEDR_local') call cedr_sl_run_local(minq, maxq, nets, nete, scalar_q_bounds, limiter_option) - ! Barrier needed to protect edge_g buffers use in CEDR. + ! Barrier needed to protect edge_g buffers used in CEDR. #if (defined HORIZ_OPENMP) !$omp barrier #endif @@ -350,7 +524,7 @@ subroutine calc_trajectory(elem, deriv, hvcoord, hybrid, dt, tl, & !$omp parallel do private(k) #endif do k = 1, nlev - call ALE_departure_from_gll(dep_points_all(:,:,k,ie), & + call ALE_departure_from_gll(dep_points_all(:,:,:,k,ie), dep_points_ndim, & elem(ie)%derived%vstar(:,:,:,k), elem(ie), dt, normalize=is_sphere) end do end do @@ -361,7 +535,7 @@ end subroutine calc_trajectory !SUBROUTINE ALE_RKDSS ! AUTHOR: CHRISTOPH ERATH, MARK TAYLOR, 06. December 2012 ! - ! DESCRIPTION: ! create a runge kutta taylor serios mixture to calculate the departure grid + ! DESCRIPTION: ! create a runge kutta taylor series mixture to calculate the departure grid ! ! CALLS: ! INPUT: @@ -372,7 +546,6 @@ end subroutine calc_trajectory ! this will calculate the velocity at time t+1/2 along the trajectory s(t) given the velocities ! at the GLL points at time t and t+1 using a second order time accurate formulation. subroutine ALE_RKdss(elem, nets, nete, hy, deriv, dt, tl, independent_time_steps) - use derivative_mod, only : derivative_t, ugradv_sphere use edgetype_mod, only : EdgeBuffer_t use bndry_mod, only : bndry_exchangev use kinds, only : real_kind @@ -397,22 +570,20 @@ subroutine ALE_RKdss(elem, nets, nete, hy, deriv, dt, tl, independent_time_steps ! RK-SSP 2 stage 2nd order: ! x*(t+1) = x(t) + U(x(t),t) dt - ! x(t+1) = x(t) + 1/2 ( U(x*(t+1),t+1) + U(x(t),t) ) dt + ! x(t+1) = x(t) + 1/2 ( U(x*(t+1),t+1) + U(x(t),t) ) dt ! apply taylor series: - ! U(x*(t+1),t+1) = U(x(t),t+1) + (x*(t+1)-x(t)) gradU(x(t),t+1) - ! - ! x(t+1) = x(t) + 1/2 ( U(x(t),t+1) + (x*(t+1)-x(t)) gradU(x(t),t+1) + U(x(t),t) ) dt - ! (x(t+1) - x(t)) / dt = 1/2 ( U(x(t),t+1) + (x*(t+1)-x(t)) gradU(x(t),t+1) + U(x(t),t) ) - ! (x(t+1) - x(t)) / dt = 1/2 ( U(x(t),t+1) + U(x(t),t) + (x*(t+1)-x(t)) gradU(x(t),t+1) ) - ! (x(t+1) - x(t)) / dt = 1/2 ( U(x(t),t+1) + U(x(t),t) + U(x(t),t) dt gradU(x(t),t+1) ) + ! U(x*(t+1),t+1) = U(x(t),t+1) + (x*(t+1)-x(t)) gradU(x(t),t+1) ! + ! x(t+1) = x(t) + 1/2 ( U(x(t),t+1) + (x*(t+1)-x(t)) gradU(x(t),t+1) + U(x(t),t) ) dt + ! (x(t+1) - x(t)) / dt = 1/2 ( U(x(t),t+1) + (x*(t+1)-x(t)) gradU(x(t),t+1) + U(x(t),t) ) + ! = 1/2 ( U(x(t),t+1) + U(x(t),t) + (x*(t+1)-x(t)) gradU(x(t),t+1) ) + ! = 1/2 ( U(x(t),t+1) + U(x(t),t) + U(x(t),t) dt gradU(x(t),t+1) ) ! - ! (x(t+1)-x(t))/dt = 1/2(U(x(t),t+1) + U(x(t),t) + dt U(x(t),t) gradU(x(t),t+1)) + ! => (x(t+1)-x(t))/dt = 1/2 (U(x(t),t+1) + U(x(t),t) + dt U(x(t),t) gradU(x(t),t+1)) ! ! suppose dt = -ts (we go backward) - ! (x(t-ts)-x(t))/-ts = 1/2( U(x(t),t-ts)+U(x(t),t)) - ts 1/2 U(x(t),t) gradU(x(t),t-ts) - ! - ! x(t-ts) = x(t)) -ts * [ 1/2( U(x(t),t-ts)+U(x(t),t)) - ts 1/2 U(x(t),t) gradU(x(t),t-ts) ] + ! (x(t-ts)-x(t))/-ts = 1/2 (U(x(t),t-ts)+U(x(t),t)) - ts 1/2 U(x(t),t) gradU(x(t),t-ts) + ! x(t-ts) = x(t)) - ts * [1/2 (U(x(t),t-ts)+U(x(t),t)) - ts 1/2 U(x(t),t) gradU(x(t),t-ts)] nlyr = 2*nlev if (independent_time_steps) nlyr = nlyr + nlev @@ -451,7 +622,7 @@ subroutine ALE_RKdss(elem, nets, nete, hy, deriv, dt, tl, independent_time_steps do ie = nets,nete call edgeVunpack_nlyr(edge_g,elem(ie)%desc,elem(ie)%derived%vstar,2*nlev,0,nlyr) if (independent_time_steps) then - call edgeVunpack_nlyr(edge_g,elem(ie)%desc,elem(ie)%derived%divdp,nlevp,2*nlev,nlyr) + call edgeVunpack_nlyr(edge_g,elem(ie)%desc,elem(ie)%derived%divdp,nlev,2*nlev,nlyr) end if end do @@ -460,8 +631,7 @@ subroutine ALE_RKdss(elem, nets, nete, hy, deriv, dt, tl, independent_time_steps #endif end subroutine ALE_RKdss - subroutine write_velocity_data(elem, nets, nete, hy, deriv, dt, tl) - use derivative_mod, only : derivative_t, ugradv_sphere + subroutine write_velocity_data(elem, nets, nete, hy, dt, tl) use edgetype_mod, only : EdgeBuffer_t use bndry_mod, only : bndry_exchangev use kinds, only : real_kind @@ -474,7 +644,6 @@ subroutine write_velocity_data(elem, nets, nete, hy, deriv, dt, tl) integer , intent(in) :: nets integer , intent(in) :: nete type (hybrid_t) , intent(in) :: hy - type (derivative_t) , intent(in) :: deriv real (kind=real_kind), intent(in) :: dt type (TimeLevel_t) , intent(in) :: tl @@ -507,7 +676,7 @@ end subroutine write_velocity_data ! ! OUTPUT: !-----------------------------------------------------------------------------------! - subroutine ALE_departure_from_gll(acart, vstar, elem, dt, normalize) + subroutine ALE_departure_from_gll(acart, ndim, vstar, elem, dt, normalize) use physical_constants, only : scale_factor use coordinate_systems_mod, only : spherical_polar_t, cartesian3D_t use time_mod, only : timelevel_t @@ -517,14 +686,15 @@ subroutine ALE_departure_from_gll(acart, vstar, elem, dt, normalize) implicit none - type (cartesian3D_t) ,intent(out) :: acart(np,np) + integer ,intent(in) :: ndim + real (kind=real_kind) ,intent(out) :: acart(ndim,np,np) real (kind=real_kind) ,intent(in) :: vstar(np,np,2) type (element_t) ,intent(in) :: elem real (kind=real_kind) ,intent(in) :: dt - logical, intent(in) :: normalize - - integer :: i,j + logical ,intent(in) :: normalize + integer :: i,j, d + type (cartesian3D_t) :: c3d real (kind=real_kind) :: uxyz (np,np,3), norm ! convert velocity from lat/lon to cartesian 3D @@ -539,16 +709,14 @@ subroutine ALE_departure_from_gll(acart, vstar, elem, dt, normalize) ! crude, 1st order accurate approximation. to be improved do i=1,np do j=1,np - call sphere2cart(elem%spherep(i,j), acart(i,j)) - acart(i,j)%x = acart(i,j)%x - dt*uxyz(i,j,1)/scale_factor - acart(i,j)%y = acart(i,j)%y - dt*uxyz(i,j,2)/scale_factor - acart(i,j)%z = acart(i,j)%z - dt*uxyz(i,j,3)/scale_factor + call sphere2cart(elem%spherep(i,j), c3d) + acart(1,i,j) = c3d%x - dt*uxyz(i,j,1)/scale_factor + acart(2,i,j) = c3d%y - dt*uxyz(i,j,2)/scale_factor + acart(3,i,j) = c3d%z - dt*uxyz(i,j,3)/scale_factor if (normalize) then - norm = sqrt(acart(i,j)%x*acart(i,j)%x + acart(i,j)%y*acart(i,j)%y + & - acart(i,j)%z*acart(i,j)%z) - acart(i,j)%x = acart(i,j)%x / norm - acart(i,j)%y = acart(i,j)%y / norm - acart(i,j)%z = acart(i,j)%z / norm + norm = sqrt(acart(1,i,j)*acart(1,i,j) + acart(2,i,j)*acart(2,i,j) + & + acart(3,i,j)*acart(3,i,j)) + acart(1:3,i,j) = acart(1:3,i,j)/norm end if enddo enddo @@ -801,7 +969,8 @@ subroutine calc_vertically_lagrangian_levels( & ! x1 - x0 = dt u(p0,t0) + O(dt^2) ! z1 - z0 = dt w(p0,t0) + O(dt^2) ! z1 = z0 + dt/2 (w(p0,t0) + w(p0,t1) + - ! dt (w_x(p0,t1) u(p0,t0) + w_z(p0,t1) w(p0,t0))) + O(dt^3) (*) + ! dt (w_x(p0,t1) u(p0,t0) + w_z(p0,t1) w(p0,t0))) + ! + O(dt^3). (*) ! Now we compute z(x0,t1). First, we need ! x0 - x1 = -dt u(p0,t0) + O(dt^2) ! and @@ -826,7 +995,7 @@ subroutine calc_vertically_lagrangian_levels( & ! - dt^2 w_x(p0,t1) u(p0,t0) + O(dt^3) ! = z0 + dt/2 (w(p0,t0) + w(p0,t1) + ! dt (-w_x(p0,t1) u(p0,t0) + w_z(p0,t1) w(p0,t0))) - ! + O(dt^3) + ! + O(dt^3). ! This is locally accurate to O(dt^3) and so globally 2nd-order ! accurate. Notably, compared with (*), this formula differs only in a ! sign. Note also that a straightforward first-order accurate formula is @@ -884,8 +1053,7 @@ subroutine calc_vertically_lagrangian_levels( & end do ! Use p0 as the reference coordinate system. p0 differs from p1 by B(eta) - ! (ps1 - ps0); dp3d already accounts for this term - ! w.r.t. derived%dp. Recall + ! (ps1 - ps0); dp3d already accounts for this term w.r.t. derived%dp. Recall ! eta_dot_dpdn = p_eta eta_dot = (A_eta p0 + B_eta ps) deta/dt, ! except that in the code eta_dot_dpdn is actually dp deta/dt rather than ! dp/deta deta/dt. eta_dot_dpdn is the motion of a pressure level excluding @@ -1158,18 +1326,1045 @@ function test_reconstruct_and_limit_dp() result(nerr) end do end function test_reconstruct_and_limit_dp - subroutine sl_unittest(par) + subroutine calc_enhanced_trajectory(elem, deriv, hvcoord, hybrid, dt, tl, & + nets, nete, nsubstep, independent_time_steps) + ! Top-level routine for new enhanced trajectory method. This new method + ! permits multiple substeps, optionally using more reference-grid velocity + ! snapshots. + + use reduction_mod, only: ParallelSum use kinds, only: iulog + + type (element_t), intent(inout) :: elem(:) + type (derivative_t), intent(in) :: deriv + type (hvcoord_t), intent(in) :: hvcoord + type (hybrid_t), intent(in) :: hybrid + real(real_kind), intent(in) :: dt + type (TimeLevel_t), intent(in) :: tl + integer, intent(in) :: nets, nete, nsubstep + logical, intent(in) :: independent_time_steps - type (parallel_t), intent(in) :: par +#ifdef HOMME_ENABLE_COMPOSE + integer :: step, ie, info, limiter_active_count + real(real_kind) :: alpha(2), dtsub + + call t_startf('SLMM_trajectory') + + call slmm_set_hvcoord(hvcoord%etai(1), hvcoord%etai(nlevp), hvcoord%etam) + + ! Set dep_points_all to level-midpoint arrival points. + call init_dep_points_all(elem, hvcoord, nets, nete, independent_time_steps) + + limiter_active_count = 0 + dtsub = dt / nsubstep + do step = 1, nsubstep + ! Fill vnode. + if (vrec%nvel == 2) then + alpha(1) = real(nsubstep - step , real_kind)/nsubstep + alpha(2) = real(nsubstep - step + 1, real_kind)/nsubstep + do ie = nets, nete + call calc_nodal_velocities(elem(ie), deriv, hvcoord, tl, & + independent_time_steps, dtsub, alpha, & + elem(ie)%derived%vstar, elem(ie)%derived%dp, & + elem(ie)%state%v(:,:,:,:,tl%np1), elem(ie)%state%dp3d(:,:,:,tl%np1), & + vnode(:,:,:,:,ie)) + end do + else + call calc_nodal_velocities_using_vrec(elem, deriv, hvcoord, tl, & + independent_time_steps, dtsub, nsubstep, step, nets, nete) + end if + + call dss_vnode(elem, nets, nete, hybrid, vnode) + + if (step == 1) then + call update_dep_points_all(independent_time_steps, dtsub, nets, nete, vnode) + else + ! Fill vdep. + call slmm_calc_v_departure(nets, nete, step, dtsub, dep_points_all, & + & dep_points_ndim, vnode, vdep, info) + + ! Using vdep, update dep_points_all to departure points. + call update_dep_points_all(independent_time_steps, dtsub, nets, nete, vdep) + end if + end do + + if (independent_time_steps) then + call interp_departure_points_to_floating_level_midpoints( & + elem, nets, nete, tl, hvcoord, dep_points_all, limiter_active_count) + ! Not needed in practice. Corner cases will be cleaned up by dss_Qdp. + !call dss_divdp(elem, nets, nete, hybrid) + if (iand(semi_lagrange_diagnostics, 1) /= 0) then + limiter_active_count = ParallelSum(limiter_active_count, hybrid) + if (limiter_active_count > 0 .and. hybrid%masterthread) then + write(iulog, '(a,i11)') 'COMPOSE> limiter_active_count', & + limiter_active_count + end if + end if + end if + + call t_stopf('SLMM_trajectory') +#endif + end subroutine calc_enhanced_trajectory + + subroutine init_dep_points_all(elem, hvcoord, nets, nete, independent_time_steps) + ! Initialize dep_points_all to the Eulerian coordinates. + + type (element_t), intent(inout) :: elem(:) + type (hvcoord_t), intent(in) :: hvcoord + integer, intent(in) :: nets, nete + logical, intent(in) :: independent_time_steps + + type (cartesian3D_t) :: c3d + integer :: ie, i, j, k + + do ie = nets, nete + do j = 1, np + do i = 1, np + call sphere2cart(elem(ie)%spherep(i,j), c3d) + dep_points_all(1,i,j,1,ie) = c3d%x + dep_points_all(2,i,j,1,ie) = c3d%y + dep_points_all(3,i,j,1,ie) = c3d%z + do k = 2, nlev + dep_points_all(1:3,i,j,k,ie) = dep_points_all(1:3,i,j,1,ie) + end do + if (independent_time_steps) then + do k = 1, nlev + dep_points_all(4,i,j,k,ie) = hvcoord%etam(k) + end do + end if + end do + end do + end do + end subroutine init_dep_points_all + + subroutine calc_nodal_velocities_using_vrec(elem, deriv, hvcoord, tl, & + independent_time_steps, dtsub, nsubstep, step, nets, nete) + + ! Wrapper to calc_nodal_velocities that orchestrates the use of the various + ! sources of velocity data. + + type (element_t), intent(in) :: elem(:) + type (derivative_t), intent(in) :: deriv + type (hvcoord_t), intent(in) :: hvcoord + type (TimeLevel_t), intent(in) :: tl + logical, intent(in) :: independent_time_steps + real(real_kind), intent(in) :: dtsub + integer, intent(in) :: nsubstep, step, nets, nete + + integer :: ie, i, k, os + real(real_kind) :: time, alpha(2), vs(np,np,2,nlev,3), dps(np,np,nlev,3) + + do ie = nets, nete + do i = 1, 2 + k = nsubstep - step + (i-1) + time = (k*vrec%t_vel(vrec%nvel))/nsubstep + os = i-1 + k = vrec%run_step(step+1-i) + if (k == 2) then + vs(:,:,:,:,os+1) = elem(ie)%derived%vstar + dps(:,:,:,os+1) = elem(ie)%derived%dp + else + vs(:,:,:,:,os+1) = vrec%vel(:,:,:,:,k-1,ie) + dps(:,:,:,os+1) = vrec%dp(:,:,:,k-1,ie) + end if + if (k == vrec%nvel) then + vs(:,:,:,:,os+2) = elem(ie)%state%v(:,:,:,:,tl%np1) + dps(:,:,:,os+2) = elem(ie)%state%dp3d(:,:,:,tl%np1) + else + vs(:,:,:,:,os+2) = vrec%vel(:,:,:,:,k,ie) + dps(:,:,:,os+2) = vrec%dp(:,:,:,k,ie) + end if + alpha(1) = (vrec%t_vel(k) - time)/(vrec%t_vel(k) - vrec%t_vel(k-1)) + alpha(2) = 1 - alpha(1) + vs(:,:,:,:,os+1) = alpha(1)*vs(:,:,:,:,os+1) + alpha(2)*vs(:,:,:,:,os+2) + dps(:,:,:, os+1) = alpha(1)*dps(:,:,:, os+1) + alpha(2)*dps(:,:,:, os+2) + end do + alpha(1) = 0 + alpha(2) = 1 + call calc_nodal_velocities(elem(ie), deriv, hvcoord, tl, & + independent_time_steps, dtsub, alpha, & + vs(:,:,:,:,1), dps(:,:,:,1), vs(:,:,:,:,2), dps(:,:,:,2), & + vnode(:,:,:,:,ie)) + end do + end subroutine calc_nodal_velocities_using_vrec + + subroutine calc_nodal_velocities(elem, deriv, hvcoord, tl, & + independent_time_steps, dtsub, alpha, v1, dp1, v2, dp2, vnode) + ! Evaluate a formula to provide an estimate of nodal velocities that + ! are use to create a 2nd-order update to the trajectory. The + ! fundamental formula for the update in position p from arrival point + ! p1 to departure point p0 is + ! p0 = p1 - dt/2 (v(p1,t0) + v(p1,t1) - dt v(p1,t1) grad v(p1,t0)). + ! Here we compute the velocity estimate at the nodes: + ! 1/2 (v(p1,t0) + v(p1,t1) - dt v(p1,t1) grad v(p1,t0)). + + type (element_t), intent(in) :: elem + type (derivative_t), intent(in) :: deriv + type (hvcoord_t), intent(in) :: hvcoord + type (TimeLevel_t), intent(in) :: tl + logical, intent(in) :: independent_time_steps + real(real_kind), intent(in) :: dtsub, alpha(2) + real(real_kind), dimension(np,np,2,nlev), intent(in) :: v1, v2 + real(real_kind), dimension(np,np,nlev), intent(in) :: dp1, dp2 + real(real_kind), intent(out) :: vnode(:,:,:,:) + + real(real_kind) :: vsph(np,np,2,nlev,2), eta_dot(np,np,nlevp,2) + integer :: t + + if (independent_time_steps) then + call calc_eta_dot_ref_mid(elem, deriv, tl, hvcoord, alpha, & + & v1, dp1, v2, dp2, eta_dot) + else + eta_dot = zero + end if + + ! Collect the horizontal nodal velocities. v1,2 are on Eulerian levels. v1 + ! is from time t1 < t2. + do t = 1, 2 + vsph(:,:,:,:,t) = (1 - alpha(t))*v1 + alpha(t)*v2 + end do + + ! Given the vertical and horizontal nodal velocities at time + ! endpoints, evaluate the velocity estimate formula, providing the + ! final horizontal and vertical velocity estimates at midpoint nodes. + call calc_vel_horiz_formula_node_ref_mid( & + & elem, deriv, hvcoord, dtsub, vsph, eta_dot, vnode) + if (independent_time_steps) then + call calc_eta_dot_formula_node_ref_mid( & + elem, deriv, hvcoord, dtsub, vsph, eta_dot, vnode) + end if + end subroutine calc_nodal_velocities + + subroutine calc_eta_dot_ref_mid(elem, deriv, tl, hvcoord, alpha, v1, dp1, v2, dp2, eta_dot) + ! Compute eta_dot at midpoint nodes at the start and end of the substep. + + type (element_t), intent(in) :: elem + type (derivative_t), intent(in) :: deriv + type (TimeLevel_t), intent(in) :: tl + type (hvcoord_t), intent(in) :: hvcoord + real(real_kind), intent(in) :: alpha(2) + real(real_kind), dimension(np,np,2,nlev), intent(in) :: v1, v2 + real(real_kind), dimension(np,np,nlev), intent(in) :: dp1, dp2 + real(real_kind), intent(out) :: eta_dot(np,np,nlevp,2) + + real(real_kind) :: vdp(np,np,2), w1(np,np) + integer :: t, k, d + + do t = 1,2 + ! eta_dot_dpdn at interface nodes. + eta_dot(:,:,1,t) = zero + do k = 1,nlev + do d = 1,2 + vdp(:,:,d) = (1 - alpha(t))*v1(:,:,d,k)*dp1(:,:,k) + & + & alpha(t) *v2(:,:,d,k)*dp2(:,:,k) + end do + w1 = divergence_sphere(vdp, deriv, elem) + eta_dot(:,:,k+1,t) = eta_dot(:,:,k,t) + w1 + end do + w1 = eta_dot(:,:,nlevp,t) + eta_dot(:,:,nlevp,t) = zero + do k = 2,nlev + eta_dot(:,:,k,t) = hvcoord%hybi(k)*w1 - eta_dot(:,:,k,t) + end do + ! Transform eta_dot_dpdn at interfaces to eta_dot at midpoints using the + ! formula + ! eta_dot = eta_dot_dpdn/(A_eta p0 + B_eta ps) + ! a= eta_dot_dpdn diff(eta)/(diff(A) p0 + diff(B) ps). + ! Compute ps. + w1 = hvcoord%hyai(1)*hvcoord%ps0 + & + & (1 - alpha(t))*sum(dp1, 3) + & + & alpha(t) *sum(dp2, 3) + do k = 1,nlev + eta_dot(:,:,k,t) = half*(eta_dot(:,:,k,t) + eta_dot(:,:,k+1,t)) & + & * (hvcoord%etai(k+1) - hvcoord%etai(k)) & + & / ( (hvcoord%hyai(k+1) - hvcoord%hyai(k))*hvcoord%ps0 & + & + (hvcoord%hybi(k+1) - hvcoord%hybi(k))*w1) + end do + end do + end subroutine calc_eta_dot_ref_mid + + subroutine calc_vel_horiz_formula_node_ref_mid( & + elem, deriv, hvcoord, dtsub, vsph, eta_dot, vnode) + + type (element_t), intent(in) :: elem + type (derivative_t), intent(in) :: deriv + type (hvcoord_t), intent(in) :: hvcoord + real(real_kind), intent(in) :: dtsub, vsph(np,np,2,nlev,2), eta_dot(np,np,nlevp,2) + real(real_kind), intent(inout) :: vnode(:,:,:,:) + + integer, parameter :: t0 = 1, t1 = 2 + + real(real_kind) :: vfsph(np,np,2), w1(np,np), w2(np,np), w3(np,np,3) + integer :: k, d, i, k1, k2 + + do k = 1, nlev + ! Horizontal terms. + vfsph = ugradv_sphere(vsph(:,:,:,k,t1), vsph(:,:,:,k,t0), deriv, elem) + vfsph = vsph(:,:,:,k,t0) + vsph(:,:,:,k,t1) - dtsub*vfsph + ! Vertical term. + do d = 1, 2 ! horiz vel dims + if (k == 1 .or. k == nlev) then + if (k == 1) then + k1 = 1; k2 = 2 + else + k1 = nlev-1; k2 = nlev + end if + w1 = (vsph(:,:,d,k2,t0) - vsph(:,:,d,k1,t0)) / & + (hvcoord%etam(k2) - hvcoord%etam(k1)) + else + do i = 1, 3 + w3(:,:,i) = hvcoord%etam(k-2+i) ! interp support + end do + w2 = hvcoord%etam(k) ! derivative at this eta value + call eval_lagrange_poly_derivative(3, w3, vsph(:,:,d,k-1:k+1,t0), w2, w1) + end if + vfsph(:,:,d) = vfsph(:,:,d) - dtsub*eta_dot(:,:,k,t1)*w1 + end do + ! Finish the formula. + vfsph = half*vfsph + ! Transform to Cartesian. + do d = 1, 3 + vnode(d,:,:,k) = sum(elem%vec_sphere2cart(:,:,d,:)*vfsph, 3) + end do + end do + end subroutine calc_vel_horiz_formula_node_ref_mid + + subroutine calc_eta_dot_formula_node_ref_mid( & + elem, deriv, hvcoord, dtsub, vsph, eta_dot, vnode) + type (element_t), intent(in) :: elem + type (derivative_t), intent(in) :: deriv + type (hvcoord_t), intent(in) :: hvcoord + real(real_kind), intent(in) :: dtsub, vsph(np,np,2,nlev,2), eta_dot(np,np,nlevp,2) + real(real_kind), intent(inout) :: vnode(:,:,:,:) + + integer, parameter :: t0 = 1, t1 = 2 + + real(real_kind) :: vfsph(np,np,2), w1(np,np), w2(np,np), w3(np,np,3), w4(np,np,3) + integer :: k, d, i, k1, k2 + + do k = 1, nlev + w2 = hvcoord%etam(k) + if (k == 1 .or. k == nlev) then + if (k == 1) then + w3(:,:,1) = hvcoord%etai(1) + w4(:,:,1) = zero + do i = 1, 2 + w3(:,:,i+1) = hvcoord%etam(i) + w4(:,:,i+1) = eta_dot(:,:,i,t0) + end do + else + do i = 1, 2 + w3(:,:,i) = hvcoord%etam(nlev-2+i) + w4(:,:,i) = eta_dot(:,:,nlev-2+i,t0) + end do + w3(:,:,3) = hvcoord%etai(nlevp) + w4(:,:,3) = zero + end if + call eval_lagrange_poly_derivative(3, w3, w4, w2, w1) + else + k1 = k-1 + k2 = k+1 + do i = 1, 3 + w3(:,:,i) = hvcoord%etam(k1-1+i) + end do + call eval_lagrange_poly_derivative(k2-k1+1, w3, eta_dot(:,:,k1:k2,t0), w2, w1) + end if + w3(:,:,1:2) = gradient_sphere(eta_dot(:,:,k,t0), deriv, elem%Dinv) + vnode(4,:,:,k) = & + half*(eta_dot(:,:,k,t0) + eta_dot(:,:,k,t1) & + & - dtsub*(vsph(:,:,1,k,t1)*w3(:,:,1) + vsph(:,:,2,k,t1)*w3(:,:,2) & + & + eta_dot(:,:,k,t1)*w1)) + end do + end subroutine calc_eta_dot_formula_node_ref_mid + + subroutine update_dep_points_all(independent_time_steps, dtsub, nets, nete, vdep) + ! Determine the departure points corresponding to the reference grid's + ! arrival midpoints. Reads and writes dep_points_all. Reads vdep. + + use physical_constants, only: scale_factor + + logical, intent(in) :: independent_time_steps + real(real_kind), intent(in) :: dtsub + integer, intent(in) :: nets, nete + real(real_kind), intent(in) :: vdep(:,:,:,:,:) + + real(real_kind) :: norm, p(3) + integer :: ie, k, j, i + + do ie = nets, nete + do k = 1, nlev + do j = 1, np + do i = 1, np + ! Update horizontal position. + p = dep_points_all(1:3,i,j,k,ie) + p = p - dtsub*vdep(1:3,i,j,k,ie)/scale_factor + if (is_sphere) then + norm = sqrt(p(1)*p(1) + p(2)*p(2) + p(3)*p(3)) + p = p/norm + end if + dep_points_all(1:3,i,j,k,ie) = p + if (independent_time_steps) then + ! Update vertical position. + dep_points_all(4,i,j,k,ie) = dep_points_all(4,i,j,k,ie) - & + & dtsub*vdep(4,i,j,k,ie) + end if + end do + end do + end do + end do + end subroutine update_dep_points_all + + subroutine interp_departure_points_to_floating_level_midpoints( & + elem, nets, nete, tl, hvcoord, dep_points_all, limcnt) + ! Determine the departure points corresponding to the vertically Lagragnian + ! grid's arrival midpoints, where the floating levels are those that evolve + ! over the course of the full tracer time step. Also compute + ! elem%derived%divdp, which holds the floating levels' dp values for later + ! use in vertical remap. + + type (element_t), intent(inout) :: elem(:) + integer, intent(in) :: nets, nete + type (hvcoord_t), intent(in) :: hvcoord + type (TimeLevel_t), intent(in) :: tl + real(real_kind), intent(inout) :: dep_points_all(:,:,:,:,:) + integer, intent(inout) :: limcnt + + real(real_kind) :: deta_ref(nlevp), w1(np,np), v1(np,np,nlev), & + & v2(np,np,nlevp), p(3) + integer :: ie, i, j, k, d + + call set_deta_tol(hvcoord) + + deta_ref(1) = hvcoord%etam(1) - hvcoord%etai(1) + do k = 2, nlev + deta_ref(k) = hvcoord%etam(k) - hvcoord%etam(k-1) + end do + deta_ref(nlevp) = hvcoord%etai(nlevp) - hvcoord%etam(nlev) + + do ie = nets, nete + ! Surface pressure. + w1 = hvcoord%hyai(1)*hvcoord%ps0 + sum(elem(ie)%state%dp3d(:,:,:,tl%np1), 3) + + ! Reconstruct Lagrangian levels at t1 on arrival column: + ! eta_arr_int = I[eta_ref_mid([0,eta_dep_mid,1])](eta_ref_int) + call limit_etam(hvcoord, deta_ref, dep_points_all(4,:,:,:,ie), v1, limcnt) + v2(:,:,1) = hvcoord%etai(1) + v2(:,:,nlevp) = hvcoord%etai(nlevp) + call eta_interp_eta(hvcoord, v1, hvcoord%etam, & + & nlevp-2, hvcoord%etai(2:nlev), v2(:,:,2:nlev)) + call eta_to_dp(hvcoord, w1, v2, elem(ie)%derived%divdp) + + ! Compute Lagrangian level midpoints at t1 on arrival column: + ! eta_arr_mid = I[eta_ref_mid([0,eta_dep_mid,1])](eta_ref_mid) + call eta_interp_eta(hvcoord, v1, hvcoord%etam, & + & nlev, hvcoord%etam, v2(:,:,1:nlev)) + dep_points_all(4,:,:,:,ie) = v2(:,:,1:nlev) + + ! Compute departure horizontal points corresponding to arrival + ! Lagrangian level midpoints: + ! p_dep_mid(eta_arr_mid) = I[p_dep_mid(eta_ref_mid)](eta_arr_mid) + do d = 1, 3 + v1 = dep_points_all(d,:,:,:,ie) + call eta_interp_horiz(hvcoord, hvcoord%etam, v1, & + & v2(:,:,1:nlev), dep_points_all(d,:,:,:,ie)) + end do + if (is_sphere) then + ! Normalize p = (x,y,z). + do k = 1, nlev + do j = 1, np + do i = 1, np + p = dep_points_all(1:3,i,j,k,ie) + p = p/sqrt(p(1)*p(1) + p(2)*p(2) + p(3)*p(3)) + dep_points_all(1:3,i,j,k,ie) = p + end do + end do + end do + end if + end do + end subroutine interp_departure_points_to_floating_level_midpoints + + subroutine set_deta_tol(hvcoord) + type (hvcoord_t), intent(in) :: hvcoord + + real(real_kind) :: deta_ave + integer :: k + + if (deta_tol >= 0) return + + ! Benign write race condition. A thread might see eta_tol < 0 and set it + ! here even as another thread does the same. But because there is no read + ! and only one value to write, the redundant writes don't matter. + + deta_ave = (hvcoord%etai(nlev+1) - hvcoord%etai(1)) / nlev + deta_tol = 10_real_kind*eps*deta_ave + end subroutine set_deta_tol + + subroutine limit_etam(hvcoord, deta_ref, eta, eta_lim, cnt) + type (hvcoord_t), intent(in) :: hvcoord + real(real_kind), intent(in) :: deta_ref(nlevp), eta(np,np,nlev) + real(real_kind), intent(out) :: eta_lim(np,np,nlev) + integer, intent(inout) :: cnt + + real(real_kind) :: deta(nlevp) + integer :: i, j, k + logical :: ok + + do j = 1, np + do i = 1, np + ! Check nonmonotonicity in eta. + ok = eta(i,j,1) - hvcoord%etai(1) >= deta_tol + if (ok) then + do k = 2, nlev + if (eta(i,j,k) - eta(i,j,k-1) < deta_tol) then + ok = .false. + exit + end if + end do + if (ok) then + ok = hvcoord%etai(nlevp) - eta(i,j,nlev) >= deta_tol + end if + end if + ! eta is monotonically increasing, so don't need to do anything + ! further. + if (ok) then + eta_lim(i,j,:) = eta(i,j,:) + cycle + end if + + deta(1) = eta(i,j,1) - hvcoord%etai(1) + do k = 2, nlev + deta(k) = eta(i,j,k) - eta(i,j,k-1) + end do + deta(nlevp) = hvcoord%etai(nlevp) - eta(i,j,nlev) + ! [0, etam(1)] and [etam(nlev),1] are half levels, but deta_tol is so + ! small there's no reason not to use it as a lower bound for these. + cnt = cnt + 1 + call deta_caas(nlevp, deta_ref, deta_tol, deta) + eta_lim(i,j,1) = hvcoord%etai(1) + deta(1) + do k = 2, nlev + eta_lim(i,j,k) = eta_lim(i,j,k-1) + deta(k) + end do + end do + end do + end subroutine limit_etam + + subroutine deta_caas(nlp, deta_ref, lo, deta) + integer, intent(in) :: nlp + real(real_kind), intent(in) :: deta_ref(nlp), lo + real(real_kind), intent(inout) :: deta(nlp) + + real(real_kind) :: nerr, w(nlp) + integer :: k + + nerr = zero + do k = 1, nlp + if (deta(k) < lo) then + nerr = nerr + (deta(k) - lo) + deta(k) = lo + w(k) = zero + else + if (deta(k) > deta_ref(k)) then + ! Only pull mass from intervals that are larger than their + ! reference value. This concentrates changes to intervals that, by + ! having a lot more mass than usual, drive other levels negative. + w(k) = deta(k) - deta_ref(k) + else + w(k) = zero + end if + end if + end do + if (nerr /= zero) deta = deta + nerr*(w/sum(w)) + end subroutine deta_caas + + subroutine linterp(n, x, y, ni, xi, yi, caller) + ! Linear interpolant: yi = I[y(x)](xi). + ! x and xi must be in ascending order. + ! xi(1) must be >= x(1) and xi(ni) must be <= x(n). + + use kinds, only: iulog + + integer, intent(in) :: n, ni + real(real_kind), intent(in) :: x(n), y(n), xi(ni) + real(real_kind), intent(out) :: yi(ni) + character(len=*), intent(in) :: caller + + integer :: k, ki + real(real_kind) :: a + + if (xi(1) < x(1) .or. xi(ni) > x(n)) then + write(iulog,*) 'x', x + write(iulog,*) 'xi', xi + call abortmp('sl_vertically_remap_tracers> linterp xi out of bounds: ' & + // trim(caller)) + end if + + k = 2 + do ki = 1, ni + do while (x(k) < xi(ki)) + k = k + 1 + end do + a = (xi(ki) - x(k-1))/(x(k) - x(k-1)) + yi(ki) = (1 - a)*y(k-1) + a*y(k) + end do + end subroutine linterp + + subroutine eta_interp_eta(hvcoord, x, y, ni, xi, yi) + type (hvcoord_t), intent(in) :: hvcoord + real(real_kind), intent(in) :: x(np,np,nlev), y(nlev) + integer, intent(in) :: ni + real(real_kind), intent(in) :: xi(ni) + real(real_kind), intent(out) :: yi(np,np,ni) + + real(real_kind) :: x01(nlev+2), y01(nlev+2) + integer :: i, j + + x01(1) = hvcoord%etai(1) + x01(nlev+2) = hvcoord%etai(nlevp) + y01(1) = hvcoord%etai(1) + y01(2:nlev+1) = y + y01(nlev+2) = hvcoord%etai(nlevp) + do j = 1, np + do i = 1, np + x01(2:nlev+1) = x(i,j,:) + call linterp(nlev+2, x01, y01, & + & ni, xi, yi(i,j,:), & + & 'eta_interp_eta') + end do + end do + end subroutine eta_interp_eta + + subroutine eta_interp_horiz(hvcoord, x, y, xi, yi) + type (hvcoord_t), intent(in) :: hvcoord + real(real_kind), intent(in) :: x(nlev), y(np,np,nlev), xi(np,np,nlev) + real(real_kind), intent(out) :: yi(np,np,nlev) + + real(real_kind) :: xbdy(nlev+2), ybdy(nlev+2) + integer :: i, j + + xbdy(1) = hvcoord%etai(1) + xbdy(2:nlev+1) = x + xbdy(nlev+2) = hvcoord%etai(nlevp) + do j = 1, np + do i = 1, np + ! Do constant interp outside of the etam support. + ybdy(1) = y(i,j,1) + ybdy(2:nlev+1) = y(i,j,:) + ybdy(nlev+2) = y(i,j,nlev) + call linterp(nlev+2, xbdy, ybdy, & + & nlev, xi(i,j,:), yi(i,j,:), & + & 'eta_interp_horiz') + end do + end do + end subroutine eta_interp_horiz + + subroutine eta_to_dp(hvcoord, ps, etai, dp) + ! e = A(e) + B(e) + ! p(e) = A(e) p0 + B(e) ps + ! = e p0 + B(e) (ps - p0) + ! a= e p0 + I[Bi(eref)](e) (ps - p0) + + type (hvcoord_t), intent(in) :: hvcoord + real(real_kind), intent(in) :: ps(np,np), etai(np,np,nlevp) + real(real_kind), intent(out) :: dp(np,np,nlev) + + real(real_kind) :: Bi(nlevp), dps + integer :: i, j, k + + do j = 1, np + do i = 1, np + call linterp(nlevp, hvcoord%etai, hvcoord%hybi, & + & nlevp, etai(i,j,:), Bi, & + & 'eta_to_dp') + dps = ps(i,j) - hvcoord%ps0 + do k = 1, nlev + dp(i,j,k) = (etai(i,j,k+1) - etai(i,j,k))*hvcoord%ps0 + & + & (Bi(k+1) - Bi(k))*dps + end do + end do + end do + end subroutine eta_to_dp + + subroutine dss_vnode(elem, nets, nete, hybrid, vnode) + type (element_t), intent(in) :: elem(:) + type (hybrid_t), intent(in) :: hybrid + integer, intent(in) :: nets, nete + real(real_kind) :: vnode(:,:,:,:,:) + + integer :: nd, nlyr, ie, k, d + + nd = size(vnode, 1) + nlyr = nd*nlev + + do ie = nets, nete + do k = 1, nlev + do d = 1, nd + vnode(d,:,:,k,ie) = vnode(d,:,:,k,ie)* & + & elem(ie)%spheremp*elem(ie)%rspheremp + end do + end do + do d = 1, nd + call edgeVpack_nlyr(edge_g, elem(ie)%desc, vnode(d,:,:,:,ie), & + & nlev, nlev*(d-1), nlyr) + end do + end do + + call t_startf('SLMM_bexchV') + call bndry_exchangeV(hybrid, edge_g) + call t_stopf('SLMM_bexchV') + + do ie = nets, nete + do d = 1, nd + call edgeVunpack_nlyr(edge_g, elem(ie)%desc, vnode(d,:,:,:,ie), & + & nlev, nlev*(d-1), nlyr) + end do + end do + +#if (defined HORIZ_OPENMP) + !$omp barrier +#endif + end subroutine dss_vnode + + subroutine dss_divdp(elem, nets, nete, hybrid) + type (element_t), intent(inout) :: elem(:) + type (hybrid_t), intent(in) :: hybrid + integer, intent(in) :: nets, nete + + integer :: ie, k + + do ie = nets, nete + do k = 1, nlev + elem(ie)%derived%divdp(:,:,k) = elem(ie)%derived%divdp(:,:,k)* & + & elem(ie)%spheremp*elem(ie)%rspheremp + end do + call edgeVpack_nlyr(edge_g, elem(ie)%desc, elem(ie)%derived%divdp, & + & nlev, 0, nlev) + end do + + call t_startf('SLMM_bexchV') + call bndry_exchangeV(hybrid, edge_g) + call t_stopf('SLMM_bexchV') + + do ie = nets, nete + call edgeVunpack_nlyr(edge_g, elem(ie)%desc, elem(ie)%derived%divdp, & + & nlev, 0, nlev) + end do + +#if (defined HORIZ_OPENMP) + !$omp barrier +#endif + end subroutine dss_divdp + + function assert(b, msg) result(nerr) + use kinds, only: iulog + + logical, intent(in) :: b + character(*), optional, intent(in) :: msg + + character(len=128) :: s integer :: nerr nerr = 0 - nerr = nerr + test_lagrange() - nerr = nerr + test_reconstruct_and_limit_dp() + if (b) return - if (nerr > 0 .and. par%masterproc) write(iulog,'(a,i3)') 'sl_unittest FAIL', nerr + s = '' + if (present(msg)) s = msg + write(iulog,'(a,a)') 'COMPOSE> sl_advection ASSERT: ', trim(s) + nerr = 1 + end function assert + + function test_linterp() result (nerr) + integer, parameter :: n = 128, ni = 111 + + real(real_kind) :: x(n), y(n), xi(ni), yi(ni), yin(n), a + integer :: k, nerr + + call random_number(x) + do k = 2, n + x(k) = x(k) + x(k-1) + end do + y = 3*x + + do k = 1, ni + a = real(k, real_kind)/(ni+1) + xi(k) = (1 - a)*x(1) + a*x(n) + end do + + call linterp(n, x, y, ni, xi, yi, 'test_linterp 1') + nerr = assert(maxval(abs( yi - 3*xi)) < 100*eps*x(n), 'linterp 1') + + call linterp(n, x, y, n, x, yin, 'test_linterp 2') + nerr = nerr + assert(maxval(abs(yin - y)) < 10*eps, 'linterp 2') + end function test_linterp + + function test_eta_to_dp(hvcoord) result(nerr) + type (hvcoord_t), intent(in) :: hvcoord + + real(real_kind) :: ps(np,np), etai(np,np,nlevp), dp1(np,np,nlev), & + & dp2(np,np,nlev) + integer :: nerr, i, j, k + + nerr = 0 + + call random_number(ps) + ps = (one + 0.2*(ps - 0.5))*hvcoord%ps0 + + do k = 1, nlev + dp1(:,:,k) = (hvcoord%hyai(k+1) - hvcoord%hyai(k))*hvcoord%ps0 + & + & (hvcoord%hybi(k+1) - hvcoord%hybi(k))*ps + end do + + ! Test that for etai_ref we get the same as the usual formula. + do j = 1, np + do i = 1, np + etai(i,j,:) = hvcoord%etai + end do + end do + call eta_to_dp(hvcoord, ps, etai, dp2) + nerr = nerr + assert(maxval(abs(dp2-dp1)) < 100*eps*maxval(dp1), 'eta_to_dp 1') + end function test_eta_to_dp + + function test_deta_caas() result(nerr) + integer, parameter :: nl = 128, nlp = nl+1 + + real(real_kind) :: deta_ref(nlp), etam_ref(nl), deta_tol, etam(nl), & + & deta(nlp) + integer :: i, k, nerr + + nerr = 0 + + call random_number(deta_ref) + deta_ref = deta_ref + 0.1 + deta_ref = deta_ref/sum(deta_ref) + + deta_tol = 10_real_kind*eps*sum(deta_ref)/size(deta_ref) + nerr = nerr + assert(deta_tol < minval(deta_ref), 'deta_caas 1') + + ! Test: Input not touched. + deta = deta_ref + call deta_caas(nlp, deta_ref, deta_tol, deta) + nerr = nerr + assert(maxval(abs(deta-deta_ref)) == zero, 'deta_caas 2') + + etam_ref(1) = deta_ref(1) + do k = 2, nl + etam_ref(k) = etam_ref(k-1) + deta_ref(k) + end do + + ! Test: Modify one etam and only adjacent intervals change beyond eps. + do i = 1, 2 + etam = etam_ref + if (i == 1) then + etam(11) = etam(11) + 1.1 + else + etam(11) = etam(11) - 13.1 + end if + deta(1) = etam(1) + do k = 2, nl + deta(k) = etam(k) - etam(k-1) + end do + deta(nlp) = one - etam(nl) + nerr = nerr + assert(minval(deta) < deta_tol, 'deta_caas 3') + call deta_caas(nlp, deta_ref, deta_tol, deta) + nerr = nerr + assert(minval(deta) == deta_tol, 'deta_caas 4') + nerr = nerr + assert(abs(sum(deta) - one) < 100*eps, 'deta_caas 5') + deta = abs(deta - deta_ref) + nerr = nerr + assert(maxval(deta(:10)) < 100*eps, 'deta_caas 6') + nerr = nerr + assert(maxval(deta(13:)) < 100*eps, 'deta_caas 7') + end do + + ! Test: Completely messed up levels. + call random_number(deta) + deta = deta - 0.5_real_kind + if (sum(deta) < 0.1) deta = deta + (0.1 - sum(deta))/nlp + deta = deta/sum(deta) + call deta_caas(nlp, deta_ref, deta_tol, deta) + nerr = nerr + assert(minval(deta) == deta_tol, 'deta_caas 8') + nerr = nerr + assert(abs(sum(deta) - one) < 1000*eps, 'deta_caas 9') + end function test_deta_caas + + function test_init_velocity_record() result(error) + integer :: dtf, drf, nsub, nvel, e, error + type (velocity_record_t) :: v + + error = 0 + dtf = 6 + drf = 2 + nsub = 3 + nvel = 4 + call init_velocity_record(1, dtf, drf, nsub, nvel, v, e) + call test(dtf, drf, nsub, nvel, v, e) + if (e > 0) error = 1 + call cleanup(v) + nvel = 3 + call init_velocity_record(1, dtf, drf, nsub, nvel, v, e) + call test(dtf, drf, nsub, nvel, v, e) + if (e > 0) error = 1 + call cleanup(v) + drf = 3 + nvel = 6 + call init_velocity_record(1, dtf, drf, nsub, nvel, v, e) + call test(dtf, drf, nsub, nvel, v, e) + if (e > 0) error = 1 + call cleanup(v) + drf = 1 + nsub = 5 + call init_velocity_record(1, dtf, drf, nsub, nvel, v, e) + call test(dtf, drf, nsub, nvel, v, e) + if (e > 0) error = 1 + call cleanup(v) + dtf = 12 + drf = 2 + nsub = 3 + nvel = -1 + call init_velocity_record(1, dtf, drf, nsub, nvel, v, e) + call test(dtf, drf, nsub, nvel, v, e) + if (e > 0) error = 1 + call cleanup(v) + nsub = 5 + nvel = 5 + call init_velocity_record(1, dtf, drf, nsub, nvel, v, e) + call test(dtf, drf, nsub, nvel, v, e) + if (e > 0) error = 1 + call cleanup(v) + dtf = 27 + drf = 3 + nsub = 51 + nvel = 99 + call init_velocity_record(1, dtf, drf, nsub, nvel, v, e) + call test(dtf, drf, nsub, nvel, v, e) + if (e > 0) error = 1 + call cleanup(v) + + contains + subroutine test(dtf, drf, nsub, nvel, v, error) + integer, intent(in) :: dtf, drf, nsub, nvel + integer, intent(inout) :: error + type (velocity_record_t), intent(in) :: v + + real(kind=real_kind) :: endslots(2), ys(dtf), a, x, y, y0, y1, & + & xsup(2), ysup(2) + integer :: n, e, i, k + + e = 0 + + if (modulo(dtf, drf) /= 0) then + print *, 'TESTING ERROR: dtf % drf == 0 is required:', dtf, drf + end if + + ! Check that t_vel is monotonically increasing. + do n = 2, v%nvel + if (v%t_vel(n) <= v%t_vel(n-1)) e = 1 + end do + + ! Check that obs_slots does not reference end points. This should not + ! happen b/c nvel <= navail and observations are uniformly spaced. + do n = 1, dtf + do i = 1, 2 + if (v%obs_slots(n,i) == 0 .or. v%obs_slots(n,i) == dtf) e = 11 + end do + end do + + ! Check that weights sum to 1. + ys = 0 + do n = 1, dtf + do i = 1, 2 + if (v%obs_slots(n,i) > 0) then + ys(v%obs_slots(n,i)) = ys(v%obs_slots(n,i)) + v%obs_wts(n,i) + end if + end do + end do + do i = 2, v%nvel-1 + if (abs(ys(i) - 1) > 1e3*eps) e = 12 + end do + + ! Test for exact interp of an affine function. + ! Observe data forward in time. + endslots(1) = tfn(0.d0) + endslots(2) = tfn(real(dtf, real_kind)) + ys(:) = 0 + do n = 1, dtf + if (modulo(n, drf) /= 0) cycle + y = tfn(real(n, real_kind)) + do i = 1, 2 + if (v%obs_slots(n,i) == -1) cycle + ys(v%obs_slots(n,i)) = ys(v%obs_slots(n,i)) + v%obs_wts(n,i)*y + end do + end do + ! Use the data backward in time. + do n = 1, nsub + ! Each segment orders the data forward in time. Thus, data are always + ! ordered forward in time but used backward. + do i = 1, 2 + k = nsub - n + (i-1) + xsup(i) = (k*v%t_vel(v%nvel))/nsub + k = v%run_step(n+1-i) + if (k == 2) then + y0 = endslots(1) + else + y0 = ys(k-1) + end if + if (k == v%nvel) then + y1 = endslots(2) + else + y1 = ys(k) + end if + ysup(i) = ((v%t_vel(k) - xsup(i))*y0 + (xsup(i) - v%t_vel(k-1))*y1) / & + & (v%t_vel(k) - v%t_vel(k-1)) + end do + do i = 0, 10 + a = real(i, real_kind)/10 + x = (1-a)*xsup(1) + a*xsup(2) + y = (1-a)*ysup(1) + a*ysup(2) + if (abs(y - tfn(x)) > 1e3*eps) e = 2 + end do + end do + + if (error > 0 .or. e > 0) then + print *, 'ERROR', error, e + print '(a,i3,a,i3,a,i3,a,i3,a,i3)', 'dtf',dtf,' drf',drf,' nsub',nsub, & + ' nvel',nvel,' v%nvel',v%nvel + print '(a,99es11.3)', ' t_vel', (v%t_vel(n),n=1,v%nvel) + do n = 1, dtf-1 + print '(3i3,2f5.2)', n, v%obs_slots(n,:), v%obs_wts(n,:) + end do + do n = 0, nsub + print '(i3,i3)', n, v%run_step(n) + end do + error = 1 + end if + end subroutine test + + function tfn(x) result(y) + real(kind=real_kind), intent(in) :: x + real(kind=real_kind) :: y + + y = 7.1*x - 11.5 + end function tfn + + subroutine cleanup(v) + type (velocity_record_t), intent(inout) :: v + deallocate(v%t_vel, v%obs_slots, v%obs_wts, v%run_step, v%vel, v%dp) + end subroutine cleanup + end function test_init_velocity_record + + subroutine sl_unittest(par, hvcoord) + use kinds, only: iulog + + type (parallel_t), intent(in) :: par + type (hvcoord_t), intent(in) :: hvcoord + + integer :: n(6) + + n(1) = test_lagrange() + n(2) = test_reconstruct_and_limit_dp() + n(3) = test_deta_caas() + n(4) = test_linterp() + n(5) = test_eta_to_dp(hvcoord) + n(6) = test_init_velocity_record() + + if (sum(n) > 0 .and. par%masterproc) then + write(iulog,'(a,6i2)') 'COMPOSE> sl_unittest FAIL ', n + end if end subroutine sl_unittest end module sl_advection diff --git a/components/homme/src/test_mod.F90 b/components/homme/src/test_mod.F90 index 361465218e5d..0f5220282cce 100644 --- a/components/homme/src/test_mod.F90 +++ b/components/homme/src/test_mod.F90 @@ -21,7 +21,7 @@ module test_mod use baroclinic_inst_mod, only: binst_init_state, jw_baroclinic use dcmip12_wrapper, only: dcmip2012_test1_1, dcmip2012_test1_2, dcmip2012_test1_3,& dcmip2012_test2_0, dcmip2012_test2_x, dcmip2012_test3, & - dcmip2012_test4_init, mtest_init, dcmip2012_test1_1_conv + dcmip2012_test4_init, mtest_init, dcmip2012_test1_conv use dcmip16_wrapper, only: dcmip2016_test1, dcmip2016_test2, dcmip2016_test3, & dcmip2016_test1_forcing, dcmip2016_test2_forcing, dcmip2016_test3_forcing, & dcmip2016_pg_init, dcmip2016_test1_pg, dcmip2016_test1_pg_forcing, dcmip2016_init @@ -68,7 +68,8 @@ subroutine set_test_initial_conditions(elem, deriv, hybrid, hvcoord, tl, nets, n case('asp_tracer'); case('baroclinic'); case('dcmip2012_test1_1'); - case('dcmip2012_test1_1_conv'); + case('dcmip2012_test1_3a_conv', 'dcmip2012_test1_3b_conv', 'dcmip2012_test1_3c_conv', & + 'dcmip2012_test1_3d_conv', 'dcmip2012_test1_3e_conv', 'dcmip2012_test1_3f_conv') case('dcmip2012_test1_2'); case('dcmip2012_test1_3'); case('dcmip2012_test2_0'); @@ -118,9 +119,10 @@ subroutine set_test_initial_conditions(elem, deriv, hybrid, hvcoord, tl, nets, n case('asp_tracer'); call asp_tracer (elem,hybrid,hvcoord,nets,nete) case('baroclinic'); call binst_init_state (elem,hybrid, nets, nete, hvcoord) case('dcmip2012_test1_1'); call dcmip2012_test1_1(elem,hybrid,hvcoord,nets,nete,0.0d0,1,timelevels) - case('dcmip2012_test1_1_conv') + case('dcmip2012_test1_3a_conv', 'dcmip2012_test1_3b_conv', 'dcmip2012_test1_3c_conv', & + 'dcmip2012_test1_3d_conv', 'dcmip2012_test1_3e_conv', 'dcmip2012_test1_3f_conv') midpoint_eta_dot_dpdn = .true. - call dcmip2012_test1_1_conv(elem,hybrid,hvcoord,nets,nete,0.0d0,1,timelevels) + call dcmip2012_test1_conv(test_case,elem,hybrid,hvcoord,deriv,nets,nete,0.0d0,1,timelevels) case('dcmip2012_test1_2'); call dcmip2012_test1_2(elem,hybrid,hvcoord,nets,nete,0.0d0,1,timelevels) case('dcmip2012_test1_3'); call dcmip2012_test1_3(elem,hybrid,hvcoord,nets,nete,0.0d0,1,timelevels,deriv) case('dcmip2012_test2_0'); call dcmip2012_test2_0(elem,hybrid,hvcoord,nets,nete) @@ -197,7 +199,9 @@ subroutine set_test_prescribed_wind(elem, deriv, hybrid, hvcoord, dt, tl, nets, ! set prescribed quantities at timelevel np1 select case(test_case) case('dcmip2012_test1_1'); call dcmip2012_test1_1(elem,hybrid,hvcoord,nets,nete,time,np1,np1) - case('dcmip2012_test1_1_conv'); call dcmip2012_test1_1_conv(elem,hybrid,hvcoord,nets,nete,time,np1,np1) + case('dcmip2012_test1_3a_conv', 'dcmip2012_test1_3b_conv', 'dcmip2012_test1_3c_conv', & + 'dcmip2012_test1_3d_conv', 'dcmip2012_test1_3e_conv', 'dcmip2012_test1_3f_conv') + call dcmip2012_test1_conv(test_case,elem,hybrid,hvcoord,deriv,nets,nete,time,np1,np1) case('dcmip2012_test1_2'); call dcmip2012_test1_2(elem,hybrid,hvcoord,nets,nete,time,np1,np1) case('dcmip2012_test1_3'); call dcmip2012_test1_3(elem,hybrid,hvcoord,nets,nete,time,np1,np1,deriv) endselect @@ -358,7 +362,9 @@ subroutine print_test_results(elem, tl, hvcoord, par) type(parallel_t), intent(in) :: par select case(test_case) - case('dcmip2012_test1_1_conv'); call dcmip2012_print_test1_conv_results(elem, tl, hvcoord, par, 1) + case('dcmip2012_test1_3a_conv', 'dcmip2012_test1_3b_conv', 'dcmip2012_test1_3c_conv', & + 'dcmip2012_test1_3d_conv', 'dcmip2012_test1_3e_conv', 'dcmip2012_test1_3f_conv') + call dcmip2012_print_test1_conv_results(test_case, elem, tl, hvcoord, par, 1) end select end subroutine print_test_results diff --git a/components/homme/src/test_src/dcmip12_wrapper.F90 b/components/homme/src/test_src/dcmip12_wrapper.F90 index bd4da6eff78a..2325137f3911 100644 --- a/components/homme/src/test_src/dcmip12_wrapper.F90 +++ b/components/homme/src/test_src/dcmip12_wrapper.F90 @@ -12,7 +12,7 @@ module dcmip12_wrapper use control_mod, only: test_case, dcmip4_moist, dcmip4_X, vanalytic use dcmip2012_test1_2_3, only: test1_advection_deformation, test1_advection_hadley, test1_advection_orography, & test2_steady_state_mountain, test2_schaer_mountain,test3_gravity_wave -use dcmip2012_test1_conv, only: test1_conv_advection_deformation +use dcmip2012_test1_conv_mod, only: test1_conv_advection, test1_conv_print_results use dcmip2012_test4, only: test4_baroclinic_wave use mtests, only: mtest_state use derivative_mod, only: derivative_t, gradient_sphere @@ -116,82 +116,6 @@ subroutine dcmip2012_test1_1(elem,hybrid,hvcoord,nets,nete,time,n0,n1) end subroutine -!_____________________________________________________________________ -subroutine dcmip2012_test1_1_conv(elem,hybrid,hvcoord,nets,nete,time,n0,n1) - - ! 3d deformational flow - - ! Use physical constants consistent with HOMME - use physical_constants, only: Rd => Rgas, p0 - - type(element_t), intent(inout), target :: elem(:) ! element array - type(hybrid_t), intent(in) :: hybrid ! hybrid parallel structure - type(hvcoord_t), intent(inout) :: hvcoord ! hybrid vertical coordinates - integer, intent(in) :: nets,nete ! start, end element index - real(rl), intent(in) :: time ! current time - integer, intent(in) :: n0,n1 ! time level indices - - logical :: initialized = .false. - - integer, parameter :: zcoords = 0 ! we are not using z coords - logical, parameter :: use_eta = .true. ! we are using hybrid eta coords - real(rl), parameter :: & - T0 = 300.d0, & ! temperature (K) - ztop = 12000.d0, & ! model top (m) - H = Rd * T0 / g ! scale height - - integer :: i,j,k,ie ! loop indices - real(rl):: lon,lat ! pointwise coordiantes - real(rl):: p,z,phis,u,v,w,T,phis_ps,ps,rho,q(4),dp,eta_dot,dp_dn ! pointwise field values - - ! set analytic vertical coordinates at t=0 - if(.not. initialized) then - !$omp barrier - !$omp master - if (hybrid%masterthread) write(iulog,*) 'initializing dcmip2012 test 1-1: 3d deformational flow' - call get_evenly_spaced_p(zi,zm,0.0_rl,ztop,H) ! get evenly spaced p levels - hvcoord%etai = exp(-zi/H) ! set eta levels from z - call set_hybrid_coefficients(hvcoord,hybrid, hvcoord%etai(1),1.0_rl)! set hybrid A and B from eta levels - call set_layer_locations(hvcoord, .true., hybrid%masterthread) - initialized = .true. - !$omp end master - !$omp barrier - endif - - ! set prescribed state at level midpoints - do ie = nets,nete; do k=1,nlev; do j=1,np; do i=1,np - lon = elem(ie)%spherep(i,j)%lon; lat = elem(ie)%spherep(i,j)%lat - z = H * log(1.0d0/hvcoord%etam(k)) - p = p0 * hvcoord%etam(k) - call test1_conv_advection_deformation(time,lon,lat,p,z,zcoords,u,v,w,T,phis,ps,rho,q(1),q(2),q(3),q(4)) - - dp = pressure_thickness(ps,k,hvcoord) - call set_state(u,v,w,T,ps,phis,p,dp,zm(k),g, i,j,k,elem(ie),n0,n1) - if(time==0) call set_tracers(q,qsize,dp,i,j,k,lat,lon,elem(ie)) - - enddo; enddo; enddo; enddo - - ! set prescribed state at level interfaces - do ie = nets,nete; do k=1,nlevp; do j=1,np; do i=1,np - lon = elem(ie)%spherep(i,j)%lon; lat = elem(ie)%spherep(i,j)%lat - z = H * log(1.0d0/hvcoord%etai(k)) - p = p0 * hvcoord%etai(k) - call test1_conv_advection_deformation(time,lon,lat,p,z,zcoords,u,v,w,T,phis,ps,rho,q(1),q(2),q(3),q(4)) - call set_state_i(u,v,w,T,ps,phis,p,zi(k),g, i,j,k,elem(ie),n0,n1) - - ! get vertical derivative of p at point i,j,k - dp_dn = ddn_hyai(k)*p0 + ddn_hybi(k)*ps - - ! get vertical eta velocity at point i,j,k - eta_dot = -g*rho*w/p0 - - ! store vertical mass flux - elem(ie)%derived%eta_dot_dpdn_prescribed(i,j,k) = eta_dot * dp_dn - - enddo; enddo; enddo; enddo - -end subroutine - !_____________________________________________________________________ subroutine dcmip2012_test1_2(elem,hybrid,hvcoord,nets,nete,time,n0,n1) @@ -333,6 +257,105 @@ subroutine dcmip2012_test1_3(elem,hybrid,hvcoord,nets,nete,time,n0,n1,deriv) end subroutine +!_____________________________________________________________________ +subroutine dcmip2012_test1_conv(test_case,elem,hybrid,hvcoord,deriv,nets,nete,time,n0,n1) + + ! 3D tracer transport tests, modified to permit good convergence testing. + + ! Use physical constants consistent with HOMME + use physical_constants, only: Rd => Rgas, p0 + + character(len=*), intent(in) :: test_case + type(element_t), intent(inout), target :: elem(:) ! element array + type(hybrid_t), intent(in) :: hybrid ! hybrid parallel structure + type(hvcoord_t), intent(inout) :: hvcoord ! hybrid vertical coordinates + type (derivative_t),intent(in) :: deriv + integer, intent(in) :: nets,nete ! start, end element index + real(rl), intent(in) :: time ! current time + integer, intent(in) :: n0,n1 ! time level indices + + logical :: initialized = .false. + + real(rl), parameter :: & + T0 = 300.d0, & ! temperature (K) + ztop = 12000.d0, & ! model top (m) + H = Rd * T0 / g ! scale height + + integer :: i,j,k,ie ! loop indices + real(rl):: lon,lat,hyai,hyam,hybi,hybm ! pointwise coordiantes + real(rl):: p,z,phis,u,v,w,T,phis_ps,ps,rho,q(5),dp,eta_dot,dp_dn ! pointwise field values + logical :: use_w + real(rl):: grad_p(np,np,2),p_i(np,np),u_i(np,np),v_i(np,np) + + ! set analytic vertical coordinates at t=0 + if (.not. initialized) then + !$omp barrier + !$omp master + if (hybrid%masterthread) then + write(iulog,*) 'initializing dcmip2012 test 3(a-e): & + &modified 3d deformational flows for convergence testing' + end if + call get_evenly_spaced_z(zi,zm,0.0_rl,ztop) ! get evenly spaced z levels + hvcoord%etai = exp(-zi/H) ! set eta levels from z + call set_hybrid_coefficients(hvcoord,hybrid,hvcoord%etai(1),1.0_rl)! set hybrid A and B from eta levels + call set_layer_locations(hvcoord, .true., hybrid%masterthread) + initialized = .true. + !$omp end master + !$omp barrier + endif + + ! set prescribed state at level midpoints + do ie = nets,nete; do k=1,nlev; do j=1,np; do i=1,np + hyam = hvcoord%hyam(k); hybm = hvcoord%hybm(k) + lon = elem(ie)%spherep(i,j)%lon; lat = elem(ie)%spherep(i,j)%lat + z = H * log(1.0d0/hvcoord%etam(k)) + p = p0 * hvcoord%etam(k) + call test1_conv_advection(test_case,time,lon,lat,hyam,hybm,p,z,u,v,w,use_w, & + & T,phis,ps,rho,q) + dp = pressure_thickness(ps,k,hvcoord) + call set_state(u,v,w,T,ps,phis,p,dp,zm(k),g, i,j,k,elem(ie),n0,n1) + if (time==0) call set_tracers(q,qsize,dp,i,j,k,lat,lon,elem(ie)) + enddo; enddo; enddo; enddo + + ! set prescribed state at level interfaces + do ie = nets,nete + do k = 1,nlevp + do j = 1,np + do i = 1,np + hyai = hvcoord%hyai(k); hybi = hvcoord%hybi(k) + lon = elem(ie)%spherep(i,j)%lon; lat = elem(ie)%spherep(i,j)%lat + z = H * log(1.0d0/hvcoord%etai(k)) + p = p0 * hvcoord%etai(k) + call test1_conv_advection(test_case,time,lon,lat,hyai,hybi,p,z,u,v,w,use_w, & + & T,phis,ps,rho,q) + call set_state_i(u,v,w,T,ps,phis,p,zi(k),g,i,j,k,elem(ie),n0,n1) + if (use_w) then + ! get vertical derivative of p at point i,j,k + dp_dn = ddn_hyai(k)*p0 + ddn_hybi(k)*ps + ! get vertical eta velocity at point i,j,k + eta_dot = -g*rho*w/p0 + ! store vertical mass flux + elem(ie)%derived%eta_dot_dpdn_prescribed(i,j,k) = eta_dot * dp_dn + else + p_i(i,j) = p + u_i(i,j) = u + v_i(i,j) = v + end if + enddo + enddo + if (.not. use_w) then + ! get vertical mass flux + grad_p = gradient_sphere(p_i,deriv,elem(ie)%Dinv) + elem(ie)%derived%eta_dot_dpdn_prescribed(:,:,k) = -u_i*grad_p(:,:,1) - v_i*grad_p(:,:,2) + end if + enddo + if (.not. use_w) then + elem(ie)%derived%eta_dot_dpdn_prescribed(:,:,1) = 0 + elem(ie)%derived%eta_dot_dpdn_prescribed(:,:,nlevp) = 0 + end if + enddo +end subroutine dcmip2012_test1_conv + !_____________________________________________________________________ subroutine dcmip2012_test2_0(elem,hybrid,hvcoord,nets,nete) @@ -814,67 +837,18 @@ subroutine set_tracers(q,nq, dp,i,j,k,lat,lon,elem) end subroutine -subroutine dcmip2012_print_test1_conv_results(elem, tl, hvcoord, par, subnum) +subroutine dcmip2012_print_test1_conv_results(test_case, elem, tl, hvcoord, par, subnum) use time_mod, only: timelevel_t use parallel_mod, only: parallel_t - use dimensions_mod, only: nelemd, nlev, qsize - use parallel_mod, only: global_shared_buf, global_shared_sum - use global_norms_mod, only: wrap_repro_sum - use physical_constants, only: Rd => Rgas, p0 + character(len=*), intent(in) :: test_case type(element_t), intent(in) :: elem(:) type(timelevel_t), intent(in) :: tl type(hvcoord_t), intent(in) :: hvcoord type(parallel_t), intent(in) :: par integer, intent(in) :: subnum - integer, parameter :: zcoords = 0 - real(rl), parameter :: & - T0 = 300.d0, & ! temperature (K) - ztop = 12000.d0, & ! model top (m) - H = Rd * T0 / g ! scale height - - real(rl) :: q(np,np,4), lon, lat, z, p, phis, u, v, w, T, phis_ps, ps, rho, time, & - a, b, reldif - integer :: ie, k, iq, i, j - - ! Set time to 0 to get the initial conditions. - time = 0._rl - - do ie = 1,nelemd - global_shared_buf(ie,:2*qsize) = 0._rl - do k = 1,nlev - z = H * log(1.0d0/hvcoord%etam(k)) - p = p0 * hvcoord%etam(k) - do j = 1,np - do i = 1,np - lon = elem(ie)%spherep(i,j)%lon - lat = elem(ie)%spherep(i,j)%lat - select case(subnum) - case (1) - call test1_conv_advection_deformation( & - time,lon,lat,p,z,zcoords,u,v,w,T,phis,ps,rho, & - q(i,j,1),q(i,j,2),q(i,j,3),q(i,j,4)) - end select - end do - end do - do iq = 1,qsize - global_shared_buf(ie,2*iq-1) = global_shared_buf(ie,2*iq-1) + & - sum(elem(ie)%spheremp*(elem(ie)%state%Q(:,:,k,iq) - q(:,:,iq))**2) - global_shared_buf(ie,2*iq) = global_shared_buf(ie,2*iq) + & - sum(elem(ie)%spheremp*q(:,:,iq)**2) - end do - end do - end do - call wrap_repro_sum(nvars=2*qsize, comm=par%comm) - if (par%masterproc) then - do iq = 1,qsize - a = global_shared_sum(2*iq-1) - b = global_shared_sum(2*iq) - reldif = sqrt(a/b) - print '(a,i2,es24.16)', 'test1_conv> Q', iq, reldif - end do - end if + call test1_conv_print_results(test_case, elem, tl, hvcoord, par, subnum) end subroutine dcmip2012_print_test1_conv_results end module dcmip12_wrapper diff --git a/components/homme/src/test_src/dcmip2012_test1_conv.F90 b/components/homme/src/test_src/dcmip2012_test1_conv.F90 deleted file mode 100644 index 274dfcacb588..000000000000 --- a/components/homme/src/test_src/dcmip2012_test1_conv.F90 +++ /dev/null @@ -1,175 +0,0 @@ -module dcmip2012_test1_conv - implicit none - -contains - -SUBROUTINE test1_conv_advection_deformation (time,lon,lat,p,z,zcoords,u,v,w,t,phis,ps,rho,q1,q2,q3,q4) -!----------------------------------------------------------------------- -! input/output params parameters at given location -!----------------------------------------------------------------------- - - ! Use physical constants consistent with HOMME - use physical_constants, only: a=>rearth0, Rd => Rgas, g, cp, pi=>dd_pi, p0 - - real(8), intent(in) :: time ! simulation time (s) - real(8), intent(in) :: lon ! Longitude (radians) - real(8), intent(in) :: lat ! Latitude (radians) - real(8), intent(in) :: z ! Height (m) - real(8), intent(inout) :: p ! Pressure (Pa) - integer, intent(in) :: zcoords ! 0 or 1 see below - real(8), intent(out) :: u ! Zonal wind (m s^-1) - real(8), intent(out) :: v ! Meridional wind (m s^-1) - real(8), intent(out) :: w ! Vertical Velocity (m s^-1) - real(8), intent(out) :: T ! Temperature (K) - real(8), intent(out) :: phis ! Surface Geopotential (m^2 s^-2) - real(8), intent(out) :: ps ! Surface Pressure (Pa) - real(8), intent(out) :: rho ! density (kg m^-3) - real(8), intent(out) :: q1 ! Tracer q1 (kg/kg) - real(8), intent(out) :: q2 ! Tracer q2 (kg/kg) - real(8), intent(out) :: q3 ! Tracer q3 (kg/kg) - real(8), intent(out) :: q4 ! Tracer q4 (kg/kg) - - ! if zcoords = 1, then we use z and output p - ! if zcoords = 0, then we use p - -!----------------------------------------------------------------------- -! test case parameters -!----------------------------------------------------------------------- - real(8), parameter :: & - tau = 12.d0 * 86400.d0, & ! period of motion 12 days - u0 = (2.d0*pi*a)/tau, & ! 2 pi a / 12 days - k0 = (10.d0*a)/tau, & ! velocity magnitude - omega0 = (2*23000.d0*pi)/tau, & ! velocity magnitude - T0 = 300.d0, & ! temperature - H = Rd * T0 / g, & ! scale height - RR = 1.d0/2.d0, & ! horizontal half width divided by 'a' - ZZ = 1000.d0, & ! vertical half width - z0 = 5000.d0, & ! center point in z - lambda0 = 5.d0*pi/6.d0, & ! center point in longitudes - lambda1 = 7.d0*pi/6.d0, & ! center point in longitudes - phi0 = 0.d0, & ! center point in latitudes - phi1 = 0.d0, & - ztop = 12000.d0 - - real(8) :: height ! The height of the model levels - real(8) :: ptop ! model top in p - real(8) :: sin_tmp, cos_tmp, sin_tmp2, cos_tmp2 ! Calculate great circle distances - real(8) :: d1, d2, r, r2, d3, d4 ! For tracer calculations - real(8) :: s, bs, s_p ! Shape function, and parameter - real(8) :: lonp ! Translational longitude, depends on time - real(8) :: ud ! Divergent part of u - real(8) :: x,y,zeta,tmp - - !--------------------------------------------------------------------- - ! HEIGHT AND PRESSURE - !--------------------------------------------------------------------- - - ! height and pressure are aligned (p = p0 exp(-z/H)) - if (zcoords .eq. 1) then - height = z - p = p0 * exp(-z/H) - else - height = H * log(p0/p) - endif - - ! model top in p - ptop = p0*exp(-ztop/H) - - !--------------------------------------------------------------------- - ! THE VELOCITIES ARE TIME DEPENDENT AND THEREFORE MUST BE UPDATED - ! IN THE DYNAMICAL CORE - !--------------------------------------------------------------------- - - ! shape function - bs = 1.0d0 - s = 1.0 + exp((ptop-p0)/(bs*ptop)) - exp((p-p0)/(bs*ptop)) - exp((ptop-p)/(bs*ptop)) - s_p = (-exp((p-p0)/(bs*ptop)) + exp((ptop-p)/(bs*ptop)))/(bs*ptop) - - ! translational longitude - lonp = lon - 2.d0*pi*time/tau - - ! zonal velocity - ud = (omega0*a) * cos(lonp) * (cos(lat)**2.0) * cos(pi*time/tau) * s_p - - u = k0*sin(lonp)*sin(lonp)*sin(2.d0*lat)*cos(pi*time/tau) + u0*cos(lat) + ud - - ! meridional velocity - v = k0*sin(2.d0*lonp)*cos(lat)*cos(pi*time/tau) - - ! vertical velocity - can be changed to vertical pressure velocity by - ! omega = -(g*p)/(Rd*T0)*w - - w = -((Rd*T0)/(g*p))*omega0*sin(lonp)*cos(lat)*cos(pi*time/tau)*s - - !----------------------------------------------------------------------- - ! TEMPERATURE IS CONSTANT 300 K - !----------------------------------------------------------------------- - t = T0 - - !----------------------------------------------------------------------- - ! PHIS (surface geopotential) - !----------------------------------------------------------------------- - phis = 0.d0 - - !----------------------------------------------------------------------- - ! PS (surface pressure) - !----------------------------------------------------------------------- - ps = p0 - - !----------------------------------------------------------------------- - ! RHO (density) - !----------------------------------------------------------------------- - rho = p/(Rd*t) - - !----------------------------------------------------------------------- - ! initialize Q, set to zero - !----------------------------------------------------------------------- - ! q = 0.d0 - - !----------------------------------------------------------------------- - ! initialize tracers - !----------------------------------------------------------------------- - ! tracer 1 - a C^inf tracer field for order of accuracy analysis - - x = cos(lat)*cos(lon) - y = cos(lat)*sin(lon) - zeta = sin(lat) - q1 = 0.3d0*(1.1 + sin(0.25d0*pi*x)*sin(0.3d0*pi*y)*sin(0.25d0*pi*zeta)*sin(pi*(p-ptop)/(p0-ptop))) - - ! tracer 2 - correlated with 1 - q2 = 0.9d0 - 0.8d0*q1**2 - - ! tracer 3 - slotted ellipse - - sin_tmp = sin(lat) * sin(phi0) - cos_tmp = cos(lat) * cos(phi0) - sin_tmp2 = sin(lat) * sin(phi1) - cos_tmp2 = cos(lat) * cos(phi1) - - ! great circle distance without 'a' - r = ACOS (sin_tmp + cos_tmp*cos(lon-lambda0)) - r2 = ACOS (sin_tmp2 + cos_tmp2*cos(lon-lambda1)) - d1 = min( 1.d0, (r/RR)**2 + ((height-z0)/ZZ)**2 ) - d2 = min( 1.d0, (r2/RR)**2 + ((height-z0)/ZZ)**2 ) - - ! make the ellipse - if (d1 .le. RR) then - q3 = 1.d0 - elseif (d2 .le. RR) then - q3 = 1.d0 - else - q3 = 0.1d0 - endif - - ! put in the slot - if (height .gt. z0 .and. abs(lat) .lt. 0.125d0) then - q3 = 0.1d0 - endif - - ! tracer 4: q4 is chosen so that, in combination with the other three tracer - ! fields with weight (3/10), the sum is equal to one - q4 = 1.d0 - 0.3d0*(q1+q2+q3) - -END SUBROUTINE test1_conv_advection_deformation - -end module dcmip2012_test1_conv diff --git a/components/homme/src/test_src/dcmip2012_test1_conv_mod.F90 b/components/homme/src/test_src/dcmip2012_test1_conv_mod.F90 new file mode 100644 index 000000000000..b8805e4486ee --- /dev/null +++ b/components/homme/src/test_src/dcmip2012_test1_conv_mod.F90 @@ -0,0 +1,548 @@ +module dcmip2012_test1_conv_mod + + ! Based on DCMIP 2012 tests 1-1,2,3. + + use parallel_mod, only: abortmp + ! Use physical constants consistent with HOMME + use physical_constants, only: a => rearth0, Rd => Rgas, g, cp, pi => dd_pi, p0 + + implicit none + private + + integer, parameter :: rt = 8 + + real(rt), parameter :: & + tau = 12.d0 * 86400.d0, & ! period of motion 12 days + T0 = 300.d0, & ! temperature (K) + ztop = 12000.d0, & ! model top (m) + H = Rd * T0 / g ! scale height + + ! For tracers. + real(rt), parameter :: qlon1 = 5.d0*(pi/6.d0), qlat1 = 0, & + & qlon2 = -qlon1, qlat2 = 0, & + & qsc_min = 0.1d0 + + real(rt), parameter :: zero = 0.d0, one = 1.d0, two = 2.d0 + + public :: test1_conv_advection, test1_conv_print_results + +contains + + subroutine get_nondiv2d_uv(time, lon, lat, u, v) + ! Classic 2D nondivergent flow field. + + real(rt), intent(in ) :: time, lon, lat + real(rt), intent(out) :: u, v + + real(rt), parameter :: & + u0 = (2.d0*pi*a)/tau, & ! 2 pi a / 12 days + k0 = (10.d0*a)/tau ! velocity magnitude + + real(rt) :: lonp + + ! translational longitude + lonp = lon - 2.d0*pi*time/tau + ! zonal velocity + u = k0*sin(lonp)*sin(lonp)*sin(2.d0*lat)*cos(pi*time/tau) + u0*cos(lat) + ! meridional velocity + v = k0*sin(2.d0*lonp)*cos(lat)*cos(pi*time/tau) + end subroutine get_nondiv2d_uv + + subroutine get_nondiv3d_uv(bs, pbot, ptop, zbot, ztop, ztaper, time, lon, lat, p, z, u, v, w) + real(rt), intent(in ) :: bs, pbot, ptop, zbot, ztop, ztaper, time, lon, lat, p, z + real(rt), intent(out) :: u, v, w + + real(rt), parameter :: omega0 = (2*23000.d0*pi)/tau + + real(rt) :: s, s_p, lonp, ud, c, arg + + ! This is essentially the test 1-1 flow. The key difference in this flow + ! removes the factor of 2 in ud and w cos-time factors. The 2 in the + ! original code makes trajectories not return to their initial points. + + ! Shape function in p. + if (p >= pbot .or. p <= ptop) then + s = 0 + s_p = 0 + else + c = 0.3d0 + arg = pi*(p - ptop)/(pbot - ptop) + s = c*sin(arg)**3 + s_p = (3*c*pi/(pbot - ptop))*sin(arg)**2*cos(arg) + end if + ! Translational longitude. + lonp = lon - 2.d0*pi*time/tau + ! Nondivergent 2D flow. + call get_nondiv2d_uv(time, lon, lat, u, v) + ! Taper the 2D nondiv (u,v) flow in the z direction. This does not induce + ! any w, and the 2D field remains nondivergent at each z. + u = u*ztaper + v = v*ztaper + ! Divergent flow. + ud = (omega0*a)*cos(lonp)*(cos(lat)**2.0)*cos(pi*time/tau)*s_p + u = u + ud + w = -((Rd*T0)/(g*p))*omega0*sin(lonp)*cos(lat)*cos(pi*time/tau)*s + end subroutine get_nondiv3d_uv + + function get_2d_cinf_tracer(lon, lat) result(q) + real(rt), intent(in) :: lon, lat + + real(rt) :: q + + real(rt) :: x, y, zeta + + x = cos(lat)*cos(lon) + y = cos(lat)*sin(lon) + zeta = sin(lat) + q = 1.5d0*(1 + sin(pi*x)*sin(pi*y)*sin(pi*zeta)) + end function get_2d_cinf_tracer + + subroutine ll2xyz(lon, lat, x, y, z) + ! Unit sphere. + + real(rt), intent(in) :: lon, lat + real(rt), intent(out) :: x, y, z + + real(rt) :: sinl, cosl + + sinl = sin(lat) + cosl = cos(lat) + x = cos(lon)*cosl + y = sin(lon)*cosl + z = sinl + end subroutine ll2xyz + + function great_circle_dist(lon1, lat1, lon2, lat2) result(d) + ! Unit sphere. + + real(rt), intent(in) :: lon1, lat1, lon2, lat2 + real(rt) :: d + + real(rt) xA, yA, zA, xB, yB, zB, cp1, cp2, cp3, cpnorm, dotprod + + call ll2xyz(lon1, lat1, xA, yA, zA) + call ll2xyz(lon2, lat2, xB, yB, zB) + cp1 = yA*zB - yB*zA + cp2 = xB*zA - xA*zB + cp3 = xA*yB - xB*yA + cpnorm = sqrt(cp1*cp1 + cp2*cp2 + cp3*cp3) + dotprod = xA*xB + yA*yB + zA*zB + d = atan2(cpnorm, dotprod) + end function great_circle_dist + + function q_gh(x, y, z, xi, yi, zi) result(q) + real(rt), intent(in) :: x, y, z, xi, yi, zi + real(rt) :: q + + real(rt), parameter :: h_max = 0.95d0, b = 5.d0 + real(rt) :: r2 + + r2 = (x - xi)**2 + (y - yi)**2 + (z - zi)**2 + q = h_max*exp(-b*r2) + end function q_gh + + function q_cb(r, ri) result(q) + real(rt), intent(in) :: r, ri + real(rt) :: q + + real(rt), parameter :: h_max = one + + q = 0.5d0*h_max*(1 + cos(pi*ri/r)) + end function q_cb + + function q_sc(clon_in, clat, lon, lat, up_slot) result(q) + real(rt), intent(in) :: clon_in, clat, lon, lat + logical, intent(in) :: up_slot + real(rt) :: q + + real(rt), parameter :: b = qsc_min, c = one, r = 0.5d0, & + & lon_thr = r/6.d0, lat_thr = 5*(r/12.d0) + + real(rt) :: clon, ri + + clon = clon_in + if (clon < zero) clon = clon + two*pi + + ri = great_circle_dist(lon, lat, clon, clat) + q = b + if (ri <= r) then + if (abs(lon - clon) >= lon_thr) then + q = c + return + else + if (up_slot) then + if (lat - clat < -lat_thr) then + q = c + return + end if + else + if (lat - clat > lat_thr) then + q = c + return + end if + end if + end if + end if + end function q_sc + + function get_2d_gaussian_hills(lon, lat) result(q) + real(rt), intent(in) :: lon, lat + real(rt) :: q + + real(rt) :: x1, y1, z1, x2, y2, z2, x, y, z + + call ll2xyz(qlon1, qlat1, x1, y1, z1) + call ll2xyz(qlon2, qlat2, x2, y2, z2) + call ll2xyz(lon, lat, x, y, z) + q = q_gh(x, y, z, x1, y1, z1) + q_gh(x, y, z, x2, y2, z2) + end function get_2d_gaussian_hills + + function get_2d_cosine_bells(lon, lat) result(q) + real(rt), intent(in) :: lon, lat + real(rt) :: q + + real(rt), parameter :: r = 0.5d0, b = 0.1d0, c = 0.9d0 + real(rt) :: h, ri + + h = 0 + ri = great_circle_dist(lon, lat, qlon1, qlat1) + if (ri < r) then + h = q_cb(r, ri) + else + ri = great_circle_dist(lon, lat, qlon2, qlat2) + if (ri < r) h = q_cb(r, ri) + end if + q = b + c*h + end function get_2d_cosine_bells + + function get_2d_correlated_cosine_bells(lon, lat) result(q) + real(rt), intent(in) :: lon, lat + real(rt) :: q + + real(rt), parameter :: a = -0.8d0, b = 0.9d0 + + q = get_2d_cosine_bells(lon, lat) + q = a*q + b + end function get_2d_correlated_cosine_bells + + function get_2d_slotted_cylinders(lon, lat) result(q) + real(rt), intent(in) :: lon, lat + real(rt) :: q + + q = q_sc(qlon1, qlat1, lon, lat, .true.) + if (q < 0.5d0) q = q_sc(qlon2, qlat2, lon, lat, .false.) + end function get_2d_slotted_cylinders + + subroutine test1_conv_advection_orography( & + test_minor,time,lon,lat,p,z,zcoords,cfv,hybrid_eta,hya,hyb,u,v,w,t,phis,ps,rho,q1,q2,q3,q4) + + character(len=1), intent(in) :: test_minor ! a, b, c, d, or e + real(rt), intent(in) :: time ! simulation time (s) + real(rt), intent(in) :: lon ! Longitude (radians) + real(rt), intent(in) :: lat ! Latitude (radians) + real(rt), intent(in) :: hya ! A coefficient for hybrid-eta coordinate + real(rt), intent(in) :: hyb ! B coefficient for hybrid-eta coordinate + + logical, intent(in) :: hybrid_eta ! flag to indicate whether the hybrid sigma-p (eta) coordinate is used + + real(rt), intent(out) :: p ! Pressure (Pa) + real(rt), intent(out) :: z ! Height (m) + + integer , intent(in) :: zcoords ! 0 or 1 see below + integer , intent(in) :: cfv ! 0, 1 or 2 see below + real(rt), intent(out) :: u ! Zonal wind (m s^-1) + real(rt), intent(out) :: v ! Meridional wind (m s^-1) + real(rt), intent(out) :: w ! Vertical Velocity (m s^-1) + real(rt), intent(out) :: t ! Temperature (K) + real(rt), intent(out) :: phis ! Surface Geopotential (m^2 s^-2) + real(rt), intent(out) :: ps ! Surface Pressure (Pa) + real(rt), intent(out) :: rho ! density (kg m^-3) + real(rt), intent(out) :: q1 ! Tracer q1 (kg/kg) + real(rt), intent(out) :: q2 ! Tracer q2 (kg/kg) + real(rt), intent(out) :: q3 ! Tracer q3 (kg/kg) + real(rt), intent(out) :: q4 ! Tracer q4 (kg/kg) + + real(rt), parameter :: & + u0 = 2.d0*pi*a/tau, & ! Velocity Magnitude (m/s) + alpha = pi/6.d0, & ! rotation angle (radians), 30 degrees + lambdam = 3.d0*pi/2.d0, & ! mountain longitude center point (radians) + phim = zero, & ! mountain latitude center point (radians) + h0 = 2000.d0, & ! peak height of the mountain range (m) + Rm = 3.d0*pi/4.d0, & ! mountain radius (radians) + ztop_t = 2000.d0, & ! transition layer + zbot_q = ztop_t + 500.d0, & ! bottom of tracers; below, all q = 0 + lon_offset = 0.5d0*pi, & ! longitudinal translation of std 2d test flow and qs + ! For Hadley-like flow. Multiply w and tracer vertical extent by (ztop + ! - ztop_t)/ztop to compensate for smaller domain. + tau_h = 86400.d0, & ! period of motion 1 day (in s) + z1_h = ztop_t + 1000.d0, & ! position of lower tracer bound (m) + z2_h = z1_h + 6000.d0, & ! position of upper tracer bound (m) + z0_h = 0.5d0*(z1_h+z2_h), & ! midpoint (m) + u0_h = 250.d0, & ! Zonal velocity magnitude (m/s) + ! w0_h is the main parameter to modify to make the test easier (smaller + ! w0_h) or harder (larger). + w0_h = 0.05d0, & ! Vertical velocity magnitude (m/s) + ! For 3D deformational flow. + bs_a = 1.0d0 ! shape function smoothness + + real(rt) :: r, height, zs, zetam, ztaper, rho0, z_q_shape, ptop, ptop_t, & + & c0, fl, fl_lat, gz, gz_z, fz, fz_z, delta, lambdam_t, u_topo_fac, & + & u0_topo, tau_topo + logical :: ps_timedep + + if (cfv /= 0) call abortmp('test1_conv_advection_orography does not support cfv != 0') + if (.not. hybrid_eta) call abortmp('test1_conv_advection_orography does not support !hybrid_eta') + if (zcoords /= 0) call abortmp('test1_conv_advection_orography does not support zcoords != 0') + + ! Mountain oscillation half-width (radians). + zetam = pi/14.d0 + ! Smooth mountains for very less resource-intensive convergence testing. + if (test_minor == 'c') zetam = pi/2.d0 + ! Smoother than default but still fairly rough. + if (test_minor == 'd' .or. test_minor == 'f') zetam = pi/6.d0 + + ps_timedep = test_minor == 'e' .or. test_minor == 'f' + lambdam_t = lambdam + if (ps_timedep) then + ! Move the topography to make ps depend on time. + u0_topo = u0 + tau_topo = tau + if (test_minor == 'e') then + u0_topo = u0_h + tau_topo = tau_h + end if + u_topo_fac = -u0_topo/two + lambdam_t = lambdam_t + & + & sin(pi*time/tau_topo)*(tau_topo/pi)*u_topo_fac & ! integral of u at lat = 0 + & /a ! to radians + end if + r = great_circle_dist(lambdam_t, phim, lon, lat) + if (r .lt. Rm) then + zs = (h0/2.d0)*(one+cos(pi*r/Rm))*cos(pi*r/zetam)**2.d0 + else + zs = zero + endif + if (test_minor == 'a') zs = zero + zs = -zs ! holes instead of mountains + phis = g*zs + ps = p0 * exp(-zs/H) + + p = hya*p0 + hyb*ps + height = H * log(p0/p) + z = height + + T = T0 + + rho = p/(Rd*T) + rho0 = p0/(Rd*T) + + if (z <= 0) then + ztaper = 0 + elseif (z >= ztop_t) then + ztaper = 1 + else + ztaper = (1 + cos(pi*(1 + z/ztop_t)))/2 + end if + + w = zero + + select case(test_minor) + case('z') ! currently unused + ! Solid body rotation + ! Zonal Velocity + u = u0*(cos(lat)*cos(alpha)+sin(lat)*cos(lon)*sin(alpha)) + ! Meridional Velocity + v = -u0*(sin(lon)*sin(alpha)) + u = u*ztaper + v = v*ztaper + case('b') + ! 2D nondiv flow in each layer. + call get_nondiv2d_uv(time, lon + lon_offset, lat, u, v) + u = u*ztaper + v = v*ztaper + case('a', 'c', 'd', 'f') + ! 3D nondiv flow. + ptop_t = p0*exp(-ztop_t/H) + ptop = p0*exp(-ztop/H) + call get_nondiv3d_uv(bs_a, ptop_t, ptop, ztop_t, ztop, ztaper, & + & time, lon + lon_offset, lat, p, z, u, v, w) + case('e') + ! Similar to Hadley-like flow but with more smoothness in derivatives. + u = u0_h*cos(lat)*cos(pi*time/tau_h)*ztaper + fl = cos(lat)**2 + fl_lat = -2*cos(lat)*sin(lat) + if (z <= 0) then + fz = 0 + fz_z = 0 + else + gz = pi*z/ztop + gz_z = pi/ztop + fz = -sin(gz)**3 + fz_z = -3*sin(gz)**2*cos(gz)*gz_z + end if + c0 = w0_h*(rho0/rho)*cos(pi*time/tau_h) + w = c0*(cos(lat)*fl_lat - 2*sin(lat)*fl)*fz + v = -a*c0*(cos(lat)*fl )*fz_z + case default + call abortmp('test1_conv_advection_orography: invalid case') + end select + + if (ps_timedep) then + ! Low-level solid-body rotational wind for consistency with the moving ps + ! field. + u = u + cos(pi*time/tau_topo)*u_topo_fac*(1 - ztaper)*cos(lat) + end if + + if (time > 0) then + q1 = 0; q2 = 0; q3 = 0; q4 = 0 + return + end if + + z_q_shape = 0.5d0*(1 - cos(2*pi*(z - zbot_q)/(ztop - zbot_q))) + if (z < zbot_q .or. z > ztop) z_q_shape = zero + + select case(test_minor) + case('e') + if (height < z2_h .and. height > z1_h) then + q1 = 0.5d0 * (one + cos(2.d0*pi*(z-z0_h)/(z2_h-z1_h))) + else + q1 = zero + end if + q2 = q1 * get_2d_cinf_tracer(lon, lat) + q3 = q1 * get_2d_gaussian_hills(lon - lon_offset, lat) + q4 = q1 * get_2d_cosine_bells(lon - lon_offset, lat) + + case default + q1 = z_q_shape * get_2d_gaussian_hills(lon - lon_offset, lat) + q2 = z_q_shape * get_2d_cosine_bells(lon - lon_offset, lat) + q4 = z_q_shape * get_2d_correlated_cosine_bells(lon - lon_offset, lat) + ! Tracer discontinuous in 3D. + q3 = qsc_min + delta = z2_h - z1_h + if ( (z >= z1_h .and. z <= z1_h + 0.25d0*delta) .or. & + (z >= z1_h + 0.4d0 *delta .and. z <= z2_h - 0.4d0 *delta) .or. & + (z <= z2_h .and. z >= z2_h - 0.25d0*delta)) then + q3 = get_2d_slotted_cylinders(lon - lon_offset, lat) + end if + end select + end subroutine test1_conv_advection_orography + + subroutine test1_conv_advection(test_case,time,lon,lat,hya,hyb,p,z,u,v,w,use_w,t,phis,ps,rho,q) + character(len=*), intent(in) :: test_case ! dcmip2012_test1_{3a-f}_conv + real(rt), intent(in) :: time ! simulation time (s) + real(rt), intent(in) :: lon, lat ! Longitude, latitude (radians) + real(rt), intent(in) :: hya, hyb ! Hybrid a, b coefficients + real(rt), intent(inout) :: z ! Height (m) + real(rt), intent(inout) :: p ! Pressure (Pa) + real(rt), intent(out) :: u ! Zonal wind (m s^-1) + real(rt), intent(out) :: v ! Meridional wind (m s^-1) + real(rt), intent(out) :: w ! Vertical Velocity (m s^-1) + logical , intent(out) :: use_w ! Should caller use w or instead div(u,v)? + real(rt), intent(out) :: T ! Temperature (K) + real(rt), intent(out) :: phis ! Surface Geopotential (m^2 s^-2) + real(rt), intent(out) :: ps ! Surface Pressure (Pa) + real(rt), intent(out) :: rho ! density (kg m^-3) + real(rt), intent(out) :: q(5) ! Tracer q1 (kg/kg) + + integer, parameter :: cfv = 0, zcoords = 0 + logical, parameter :: use_eta = .true. + + character(len=1) :: test_major, test_minor + + test_major = test_case(17:17) + if (test_major == '3') test_minor = test_case(18:18) + + use_w = .false. + select case(test_major) + case('3') + call test1_conv_advection_orography( & + test_minor,time,lon,lat,p,z,zcoords,cfv,use_eta,hya,hyb,u,v,w,t,phis,ps,rho, & + q(1),q(2),q(3),q(4)) + end select + end subroutine test1_conv_advection + + subroutine test1_conv_print_results(test_case, elem, tl, hvcoord, par, subnum) + use element_mod, only: element_t + use time_mod, only: timelevel_t + use hybvcoord_mod, only: hvcoord_t + use parallel_mod, only: parallel_t, pmax_1d + use dimensions_mod, only: nelemd, nlev, qsize, np + use parallel_mod, only: global_shared_buf, global_shared_sum + use global_norms_mod, only: wrap_repro_sum + use physical_constants, only: Rd => Rgas, p0 + + character(len=*), intent(in) :: test_case + type(element_t), intent(in) :: elem(:) + type(timelevel_t), intent(in) :: tl + type(hvcoord_t), intent(in) :: hvcoord + type(parallel_t), intent(in) :: par + integer, intent(in) :: subnum + + real(rt) :: q(np,np,5), lon, lat, z, p, phis, u, v, w, T, phis_ps, ps, rho, time, & + hya, hyb, a, b, reldif, linf_num(qsize), linf_den(qsize) + integer :: ie, k, iq, i, j + logical :: use_w + + ! Set time to 0 to get the initial conditions. + time = 0._rt + + linf_num = 0 + linf_den = 0 + do ie = 1,nelemd + global_shared_buf(ie,:2*qsize) = 0._rt + do k = 1,nlev + ! test1_conv_advection_orography uses these: + hya = hvcoord%hyam(k) + hyb = hvcoord%hybm(k) + ! test1_advection_deformation uses these, in which ps = p0: + p = p0 * hvcoord%etam(k) + z = H * log(1.0d0/hvcoord%etam(k)) + + ! Normwise relative errors. We weight the horizontal direction by + ! sphereme but do not weight the vertical direction; each vertical + ! level in a column has equal weight. + + do j = 1,np + do i = 1,np + lon = elem(ie)%spherep(i,j)%lon + lat = elem(ie)%spherep(i,j)%lat + select case(subnum) + case (1) + call test1_conv_advection( & + test_case,time,lon,lat,hya,hyb,p,z,u,v,w,use_w,T,phis,ps,rho,q(i,j,:)) + end select + end do + end do + + do iq = 1,qsize + global_shared_buf(ie,2*iq-1) = global_shared_buf(ie,2*iq-1) + & + sum(elem(ie)%spheremp*(elem(ie)%state%Q(:,:,k,iq) - q(:,:,iq))**2) + global_shared_buf(ie,2*iq) = global_shared_buf(ie,2*iq) + & + sum(elem(ie)%spheremp*q(:,:,iq)**2) + linf_num(iq) = max(linf_num(iq), & + maxval(abs(elem(ie)%state%Q(:,:,k,iq) - q(:,:,iq)))) + linf_den(iq) = max(linf_den(iq), & + maxval(abs(q(:,:,iq)))) + end do + end do + end do + + call wrap_repro_sum(nvars=2*qsize, comm=par%comm) + do iq = 1, qsize + linf_num(iq) = pmax_1d(linf_num(iq:iq), par) + linf_den(iq) = pmax_1d(linf_den(iq:iq), par) + end do + + if (par%masterproc) then + print '(a)', 'test1_conv> l2 linf' + do iq = 1,qsize + a = global_shared_sum(2*iq-1) + b = global_shared_sum(2*iq) + reldif = sqrt(a/b) + print '(a,i2,es24.16,es24.16)', 'test1_conv> Q', & + iq, reldif, linf_num(iq)/linf_den(iq) + end do + end if + end subroutine test1_conv_print_results + +end module dcmip2012_test1_conv_mod diff --git a/components/homme/src/theta-l/share/prim_advection_mod.F90 b/components/homme/src/theta-l/share/prim_advection_mod.F90 index f5941519adb2..e8c6b68f0f92 100644 --- a/components/homme/src/theta-l/share/prim_advection_mod.F90 +++ b/components/homme/src/theta-l/share/prim_advection_mod.F90 @@ -13,7 +13,8 @@ module prim_advection_mod use time_mod, only : TimeLevel_t use hybrid_mod, only : hybrid_t use control_mod, only : transport_alg - use sl_advection, only : prim_advec_tracers_remap_ALE, sl_init1 + use sl_advection, only : prim_advec_tracers_observe_velocity_ale, & + prim_advec_tracers_remap_ALE, sl_init1 use prim_advection_base, only: prim_advec_init2, prim_advec_init1_rk2, & prim_advec_tracers_remap_rk2 @@ -40,6 +41,16 @@ subroutine Prim_Advec_Init1(par, elem) end subroutine Prim_Advec_Init1 + subroutine Prim_Advec_Tracers_observe_velocity(elem, tl, n, nets, nete) + type (element_t) , intent(inout) :: elem(:) + type (TimeLevel_t) , intent(in ) :: tl + integer , intent(in ) :: n ! step in 1:dt_tracer_factor + integer , intent(in ) :: nets + integer , intent(in ) :: nete + + if (transport_alg /= 0) call Prim_Advec_Tracers_observe_velocity_ALE(elem, tl, n, nets, nete) + end subroutine Prim_Advec_Tracers_observe_velocity + subroutine Prim_Advec_Tracers_remap( elem , deriv , hvcoord , hybrid , dt , tl , nets , nete ) implicit none type (element_t) , intent(inout) :: elem(:) diff --git a/components/homme/src/theta-l/share/prim_state_mod.F90 b/components/homme/src/theta-l/share/prim_state_mod.F90 index 75c6089f62a5..835dfd2ddf38 100644 --- a/components/homme/src/theta-l/share/prim_state_mod.F90 +++ b/components/homme/src/theta-l/share/prim_state_mod.F90 @@ -793,7 +793,7 @@ subroutine prim_printstate(elem, tl,hybrid,hvcoord,nets,nete) write(iulog,100) "(E-E0)/E0 ",(TOTE(2)-TOTE0)/TOTE0 do q=1,qsize if(Qmass0(q)>0.0D0) then - write(iulog,'(a,E23.15,a,i1)') "(Q-Q0)/Q0 ",(Qmass(q,2)-Qmass0(q))/Qmass0(q)," Q",q + write(iulog,'(a,E23.15,a,i2)') "(Q-Q0)/Q0 ",(Qmass(q,2)-Qmass0(q))/Qmass0(q)," Q",q end if enddo endif diff --git a/components/homme/src/theta-l_kokkos/CMakeLists.txt b/components/homme/src/theta-l_kokkos/CMakeLists.txt index a585090ea76a..191e3821cdd7 100644 --- a/components/homme/src/theta-l_kokkos/CMakeLists.txt +++ b/components/homme/src/theta-l_kokkos/CMakeLists.txt @@ -119,7 +119,7 @@ MACRO(THETAL_KOKKOS_SETUP) ${TEST_SRC_DIR}/baroclinic_inst_mod.F90 ${TEST_SRC_DIR}/dcmip12_wrapper.F90 ${TEST_SRC_DIR}/dcmip16_wrapper.F90 - ${TEST_SRC_DIR}/dcmip2012_test1_conv.F90 + ${TEST_SRC_DIR}/dcmip2012_test1_conv_mod.F90 ${TEST_SRC_DIR}/dcmip2012_test1_2_3.F90 ${TEST_SRC_DIR}/dcmip2012_test4.F90 ${TEST_SRC_DIR}/dcmip2016-baroclinic.F90 @@ -161,6 +161,8 @@ MACRO(THETAL_KOKKOS_SETUP) ${SRC_SHARE_DIR}/cxx/ComposeTransport.cpp ${SRC_SHARE_DIR}/cxx/ComposeTransportImplGeneral.cpp ${SRC_SHARE_DIR}/cxx/ComposeTransportImplTrajectory.cpp + ${SRC_SHARE_DIR}/cxx/ComposeTransportImplEnhancedTrajectory.cpp + ${SRC_SHARE_DIR}/cxx/ComposeTransportImplEnhancedTrajectoryTests.cpp ${SRC_SHARE_DIR}/cxx/ComposeTransportImplVerticalRemap.cpp ${SRC_SHARE_DIR}/cxx/ComposeTransportImplHypervis.cpp ${SRC_SHARE_DIR}/cxx/ComposeTransportImplTest2D.cpp diff --git a/components/homme/src/theta-l_kokkos/config.h.cmake.in b/components/homme/src/theta-l_kokkos/config.h.cmake.in index 7e378c1b795c..5215d7bc6415 100644 --- a/components/homme/src/theta-l_kokkos/config.h.cmake.in +++ b/components/homme/src/theta-l_kokkos/config.h.cmake.in @@ -77,3 +77,7 @@ /* Detect whether COMPOSE passive tracer transport is enabled */ #cmakedefine HOMME_ENABLE_COMPOSE + +/* For just-in-time compilation (e.g., SYCL compilers), disable timers at the */ +/* first prim_run level when nstep == 1. */ +#cmakedefine DISABLE_TIMERS_IN_FIRST_STEP diff --git a/components/homme/src/theta-l_kokkos/cxx/prim_advance_exp.cpp b/components/homme/src/theta-l_kokkos/cxx/prim_advance_exp.cpp index f0c0c1c728a2..8f7fc13d3155 100644 --- a/components/homme/src/theta-l_kokkos/cxx/prim_advance_exp.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/prim_advance_exp.cpp @@ -26,6 +26,9 @@ void ttype7_imex_timestep (const TimeLevel& tl, const Real dt, const Real eta_av void ttype9_imex_timestep (const TimeLevel& tl, const Real dt, const Real eta_ave_w); void ttype10_imex_timestep(const TimeLevel& tl, const Real dt, const Real eta_ave_w); +// Prescribed-wind F90-C++ bridge. Test inputs are all implemented in F90. +extern "C" void set_prescribed_wind_f_bridge(int n0, int np1, int nstep, Real dt); + // -------------- IMPLEMENTATIONS -------------- // void prim_advance_exp (TimeLevel& tl, const Real dt, const bool compute_diagnostics) @@ -72,10 +75,12 @@ void prim_advance_exp (TimeLevel& tl, const Real dt, const bool compute_diagnost } #if !defined(CAM) && !defined(SCREAM) - // If prescribed wind, the dynamics was set explicitly in - // prim_driver_mod::prim_run_subcycle; skip time-integration. - if (params.prescribed_wind) + // If prescribed wind, set the dynamics explicitly and skip time-integration. + if (params.prescribed_wind) { + set_prescribed_wind_f_bridge(tl.n0, tl.np1, tl.nstep, dt); + GPTLstop("tl-ae prim_advance_exp"); return; + } #endif switch (params.time_step_type) { diff --git a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 index eae8544ca865..4e49c5c06abc 100644 --- a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 +++ b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 @@ -11,7 +11,11 @@ module prim_driver_mod use prim_driver_base, only : deriv1, smooth_topo_datasets use prim_cxx_driver_base, only : prim_init1, prim_finalize use physical_constants, only : scale_factor, laplacian_rigid_factor - + use hybrid_mod, only : hybrid_t + use hybvcoord_mod, only : hvcoord_t + use derivative_mod, only : derivative_t + use time_mod, only : timelevel_t + implicit none public :: prim_init2 @@ -23,10 +27,19 @@ module prim_driver_mod public :: prim_init_ref_states_views public :: prim_init_diags_views + type, private :: PrescribedWind_t + type (element_t), pointer :: elem(:) + type (hybrid_t) :: hybrid + type (hvcoord_t) :: hvcoord + type (derivative_t) :: deriv + integer :: nets, nete + end type PrescribedWind_t + + type (PrescribedWind_t), private :: prescribed_wind_args + contains subroutine prim_init2(elem, hybrid, nets, nete, tl, hvcoord) - use hybrid_mod, only : hybrid_t use hybvcoord_mod, only : hvcoord_t use time_mod, only : timelevel_t use prim_driver_base, only : deriv1, prim_init2_base => prim_init2 @@ -46,10 +59,6 @@ subroutine prim_init2(elem, hybrid, nets, nete, tl, hvcoord) ! Call the base version of prim_init2 call prim_init2_base(elem,hybrid,nets,nete,tl,hvcoord) - if (prescribed_wind == 1) then - call init_standalone_test(elem,deriv1,hybrid,hvcoord,tl,nets,nete) - end if - ! Init the c data structures call prim_create_c_data_structures(tl,hvcoord,elem(1)%mp) @@ -61,6 +70,10 @@ subroutine prim_init2(elem, hybrid, nets, nete, tl, hvcoord) ! Initialize dp3d from ps_v call initialize_dp3d_from_ps_c () + + if (prescribed_wind == 1) then + call init_standalone_test(elem,deriv1,hybrid,hvcoord,tl,nets,nete) + end if end subroutine prim_init2 subroutine prim_create_c_data_structures (tl, hvcoord, mp) @@ -459,8 +472,8 @@ subroutine prim_run_subcycle(elem, hybrid, nets, nete, dt, single_column, tl, hv elem_derived_FPHI, elem_derived_FQ) call t_stopf('push_to_cxx') end if - if (prescribed_wind == 1) then ! standalone Homme - call set_prescribed_wind_f(elem,deriv1,hybrid,hvcoord,dt,tl,nets,nete) + if (prescribed_wind == 1) then + call init_prescribed_wind_subcycle(elem,nets,nete,tl) end if call prim_run_subcycle_c(dt,nstep_c,nm1_c,n0_c,np1_c,nextOutputStep,nsplit_iteration) @@ -614,7 +627,7 @@ subroutine init_standalone_test(elem,deriv,hybrid,hvcoord,tl,nets,nete) use element_mod, only : element_t use derivative_mod, only : derivative_t #if !defined(CAM) && !defined(SCREAM) - use test_mod, only : set_prescribed_wind + use test_mod, only : set_test_initial_conditions #endif type (element_t), intent(inout), target :: elem(:) @@ -626,11 +639,18 @@ subroutine init_standalone_test(elem,deriv,hybrid,hvcoord,tl,nets,nete) integer , intent(in) :: nete #if !defined(CAM) && !defined(SCREAM) - real(kind=real_kind) :: dt, eta_ave_w - - dt = 0 ! value unused in initialization - eta_ave_w = 0 ! same - call set_prescribed_wind(elem,deriv,hybrid,hvcoord,dt,tl,nets,nete,eta_ave_w) + ! Already called in prim_driver_base::prim_init2: + ! call set_test_initial_conditions(elem,deriv,hybrid,hvcoord,tl,nets,nete) + ! Also already taken care of: + ! call push_test_state_to_c_wrapper() + + ! Save arguments for the C++-F90 bridge for prescribed winds. + prescribed_wind_args%elem => elem + prescribed_wind_args%hybrid = hybrid + prescribed_wind_args%hvcoord = hvcoord + prescribed_wind_args%deriv = deriv + prescribed_wind_args%nets = nets + prescribed_wind_args%nete = nete #endif end subroutine init_standalone_test @@ -655,8 +675,77 @@ subroutine compute_test_forcing_f(elem,hybrid,hvcoord,nt,ntQ,dt,nets,nete,tl) #endif end subroutine compute_test_forcing_f + subroutine push_test_state_to_c_wrapper() +#if !defined(CAM) && !defined(SCREAM) + use iso_c_binding, only : c_ptr, c_loc + use perf_mod, only : t_startf, t_stopf + use theta_f2c_mod, only : push_test_state_to_c + use element_state, only : elem_state_v, elem_state_w_i, elem_state_vtheta_dp, & + elem_state_phinh_i, elem_state_dp3d, elem_state_ps_v, & + elem_derived_eta_dot_dpdn, elem_derived_vn0 + + type (c_ptr) :: elem_state_v_ptr, elem_state_w_i_ptr, elem_state_vtheta_dp_ptr, elem_state_phinh_i_ptr + type (c_ptr) :: elem_state_dp3d_ptr, elem_state_Qdp_ptr, elem_state_Q_ptr, elem_state_ps_v_ptr + type (c_ptr) :: elem_derived_eta_dot_dpdn_ptr, elem_derived_vn0_ptr + + call t_startf('push_to_cxx') + elem_state_v_ptr = c_loc(elem_state_v) + elem_state_w_i_ptr = c_loc(elem_state_w_i) + elem_state_vtheta_dp_ptr = c_loc(elem_state_vtheta_dp) + elem_state_phinh_i_ptr = c_loc(elem_state_phinh_i) + elem_state_dp3d_ptr = c_loc(elem_state_dp3d) + elem_state_ps_v_ptr = c_loc(elem_state_ps_v) + elem_derived_vn0_ptr = c_loc(elem_derived_vn0) + elem_derived_eta_dot_dpdn_ptr = c_loc(elem_derived_eta_dot_dpdn) + call push_test_state_to_c(elem_state_ps_v_ptr, elem_state_dp3d_ptr, & + elem_state_vtheta_dp_ptr, elem_state_phinh_i_ptr, elem_state_v_ptr, & + elem_state_w_i_ptr, elem_derived_eta_dot_dpdn_ptr, elem_derived_vn0_ptr) + call t_stopf('push_to_cxx') +#endif + end subroutine push_test_state_to_c_wrapper + + subroutine init_prescribed_wind_subcycle(elem, nets, nete, tl) + ! Set the derived values used in tracer transport on the F90 side even + ! though most of the work is done on the C++ side. This is needed because + ! set_prescribed_wind accumulates certain derived quantities during + ! prim_advance_exp that get repeatedly copied from F90 to C++. Here we + ! initialize values for accumulation. + ! In summary: Call this before entering the prim_run_subcycle loop. + + use prim_driver_base, only: set_tracer_transport_derived_values + + type (element_t), intent(inout) :: elem(:) + integer, intent(in) :: nets, nete + type (timelevel_t) :: tl + + call set_tracer_transport_derived_values(elem, nets, nete, tl) + end subroutine init_prescribed_wind_subcycle + + subroutine set_prescribed_wind_f_bridge(n0, np1, nstep, dt) bind(c) + ! This routine is called from the C++ prim_advance_exp implementation inside + ! the prim_run_subcycle loop. + + use iso_c_binding, only: c_int, c_double + + integer(c_int), value, intent(in) :: n0, np1, nstep + real(c_double), value, intent(in) :: dt + + type (TimeLevel_t) :: tl + + ! Only these fields need to be valid. + tl%n0 = n0+1 + tl%np1 = np1+1 + tl%nstep = nstep + + call set_prescribed_wind_f(prescribed_wind_args%elem, prescribed_wind_args%deriv, & + prescribed_wind_args%hybrid, prescribed_wind_args%hvcoord, dt, tl, & + prescribed_wind_args%nets, prescribed_wind_args%nete) + end subroutine set_prescribed_wind_f_bridge + subroutine set_prescribed_wind_f(elem,deriv,hybrid,hvcoord,dt,tl,nets,nete) - use iso_c_binding, only : c_ptr, c_loc + ! Here we finally can compute the prescribed wind in F90 and then push the + ! data to C++. + use hybrid_mod, only : hybrid_t use hybvcoord_mod, only : hvcoord_t use time_mod, only : timelevel_t @@ -664,12 +753,7 @@ subroutine set_prescribed_wind_f(elem,deriv,hybrid,hvcoord,dt,tl,nets,nete) use derivative_mod, only : derivative_t #if !defined(CAM) && !defined(SCREAM) use control_mod, only : qsplit - use perf_mod, only : t_startf, t_stopf - use theta_f2c_mod, only : push_test_state_to_c use test_mod, only : set_prescribed_wind - use element_state, only : elem_state_v, elem_state_w_i, elem_state_vtheta_dp, & - elem_state_phinh_i, elem_state_dp3d, elem_state_ps_v, & - elem_derived_eta_dot_dpdn, elem_derived_vn0 #endif type (element_t), intent(inout), target :: elem(:) @@ -683,39 +767,17 @@ subroutine set_prescribed_wind_f(elem,deriv,hybrid,hvcoord,dt,tl,nets,nete) #if !defined(CAM) && !defined(SCREAM) type (hvcoord_t) :: hv - type (c_ptr) :: elem_state_v_ptr, elem_state_w_i_ptr, elem_state_vtheta_dp_ptr, elem_state_phinh_i_ptr - type (c_ptr) :: elem_state_dp3d_ptr, elem_state_Qdp_ptr, elem_state_Q_ptr, elem_state_ps_v_ptr - type (c_ptr) :: elem_derived_eta_dot_dpdn_ptr, elem_derived_vn0_ptr real(kind=real_kind) :: eta_ave_w ! We need to set up an hvcoord_t that can be passed as intent(inout), even ! though at this point, it won't be changed in the set_prescribed_wind call. - hv%ps0 = hvcoord%ps0 - hv%hyai = hvcoord%hyai - hv%hyam = hvcoord%hyam - hv%hybi = hvcoord%hybi - hv%hybm = hvcoord%hybm - hv%etam = hvcoord%etam - hv%etai = hvcoord%etai - hv%dp0 = hvcoord%dp0 + hv = hvcoord eta_ave_w = 1d0/qsplit call set_prescribed_wind(elem,deriv,hybrid,hv,dt,tl,nets,nete,eta_ave_w) - call t_startf('push_to_cxx') - elem_state_v_ptr = c_loc(elem_state_v) - elem_state_w_i_ptr = c_loc(elem_state_w_i) - elem_state_vtheta_dp_ptr = c_loc(elem_state_vtheta_dp) - elem_state_phinh_i_ptr = c_loc(elem_state_phinh_i) - elem_state_dp3d_ptr = c_loc(elem_state_dp3d) - elem_state_ps_v_ptr = c_loc(elem_state_ps_v) - elem_derived_vn0_ptr = c_loc(elem_derived_vn0) - elem_derived_eta_dot_dpdn_ptr = c_loc(elem_derived_eta_dot_dpdn) - call push_test_state_to_c(elem_state_ps_v_ptr, elem_state_dp3d_ptr, & - elem_state_vtheta_dp_ptr, elem_state_phinh_i_ptr, elem_state_v_ptr, & - elem_state_w_i_ptr, elem_derived_eta_dot_dpdn_ptr, elem_derived_vn0_ptr) - call t_stopf('push_to_cxx') + call push_test_state_to_c_wrapper() #endif end subroutine set_prescribed_wind_f diff --git a/components/homme/test/reg_test/namelists/thetah-sl-test11conv-r0t1-cdr30-rrm.nl b/components/homme/test/reg_test/namelists/thetah-sl-test11conv-r0t1-cdr30-rrm.nl index 564ea8be17bd..5b7e987bd0b7 100644 --- a/components/homme/test/reg_test/namelists/thetah-sl-test11conv-r0t1-cdr30-rrm.nl +++ b/components/homme/test/reg_test/namelists/thetah-sl-test11conv-r0t1-cdr30-rrm.nl @@ -2,7 +2,7 @@ nthreads = -1 ! use OMP_NUM_THREADS partmethod = 4 ! mesh parition method: 4 = space filling curve topology = "cube" ! mesh type: cubed sphere - test_case = "dcmip2012_test1_1_conv" ! test identifier + test_case = "dcmip2012_test1_3a_conv" ! test identifier prescribed_wind = 1 mesh_file = 'mountain_10_x2.g' qsize = 4 ! num tracer fields diff --git a/components/homme/test/reg_test/namelists/thetah-sl-test11conv-r1t2-cdr20.nl b/components/homme/test/reg_test/namelists/thetah-sl-test11conv-r1t2-cdr20.nl index 484d3612dd07..c6a7bfb2b1cb 100644 --- a/components/homme/test/reg_test/namelists/thetah-sl-test11conv-r1t2-cdr20.nl +++ b/components/homme/test/reg_test/namelists/thetah-sl-test11conv-r1t2-cdr20.nl @@ -2,7 +2,7 @@ nthreads = -1 ! use OMP_NUM_THREADS partmethod = 4 ! mesh parition method: 4 = space filling curve topology = "cube" ! mesh type: cubed sphere - test_case = "dcmip2012_test1_1_conv" ! test identifier + test_case = "dcmip2012_test1_3a_conv" ! test identifier prescribed_wind = 1 ne = 5 ! number of elements per cube face qsize = 4 ! num tracer fields diff --git a/components/homme/test/reg_test/namelists/thetah-sl-testconv-3e.nl b/components/homme/test/reg_test/namelists/thetah-sl-testconv-3e.nl new file mode 100644 index 000000000000..d140d689ac07 --- /dev/null +++ b/components/homme/test/reg_test/namelists/thetah-sl-testconv-3e.nl @@ -0,0 +1,62 @@ +&ctl_nl + nthreads = -1 + partmethod = 4 + topology = "cube" + test_case = 'dcmip2012_test1_3e_conv' + prescribed_wind = 1 + qsize = 4 + ndays = 1 + statefreq = 240 + restartfreq = -1 + runtype = 0 + ne = 20 + integration = 'explicit' + tstep_type = 1 + smooth = 0 + nu = 1.585e13 ! nu values are irrelevant + nu_s = 1.585e13 + nu_p = 42 ! to satisfy kokkos exe + se_ftype = -1 + limiter_option = 9 + hypervis_order = 2 + hypervis_subcycle = 1 + moisture = 'dry' + theta_hydrostatic_mode = .true. + dcmip16_prec_type = 1 + dcmip16_pbl_type = -1 + transport_alg = 12 + semi_lagrange_cdr_alg = 3 + semi_lagrange_cdr_check = .false. + semi_lagrange_hv_q = 0 + semi_lagrange_nearest_point_lev = 0 + semi_lagrange_halo = 2 + dt_remap_factor = 0 + dt_tracer_factor = 4 + tstep = 200.0 + semi_lagrange_trajectory_nsubstep = 2 + semi_lagrange_trajectory_nvelocity = 3 + semi_lagrange_diagnostics = 1 + hypervis_subcycle_q = 0 + limiter_option = 9 + vert_remap_q_alg = 10 +/ +&vert_nl + vanalytic = 1 + vtop = 0.2549944 +/ +&analysis_nl + output_dir = "./movies/" + output_timeunits = 2, ! 1=days, 2=hours, 0=timesteps + output_frequency = 2, + output_varnames1 = 'ps','Q','u','v' + interp_type = 0 + output_type = 'netcdf' + num_io_procs = 16 + interp_nlon = 180 + interp_nlat = 91 + interp_gridtype = 2 +/ +&prof_inparm + profile_outpe_num = 100 + profile_single_file = .true. +/ diff --git a/components/homme/test/reg_test/run_tests/test-list.cmake b/components/homme/test/reg_test/run_tests/test-list.cmake index dbb0fc4f98e9..884a785d8433 100644 --- a/components/homme/test/reg_test/run_tests/test-list.cmake +++ b/components/homme/test/reg_test/run_tests/test-list.cmake @@ -47,7 +47,7 @@ IF (HOMME_ENABLE_COMPOSE) thetah-sl-test11conv-r1t2-cdr20.cmake thetah-sl-test11conv-r0t1-cdr30-rrm.cmake thetah-sl-dcmip16_test1pg2.cmake - ) + thetah-sl-testconv-3e.cmake) ENDIF() SET(HOMME_RUN_TESTS_DIR ${HOMME_SOURCE_DIR}/test/reg_test/run_tests) @@ -92,7 +92,9 @@ ENDIF() IF (BUILD_HOMME_THETA_KOKKOS) # Various one-off tests. IF (HOMME_ENABLE_COMPOSE) - LIST(APPEND HOMME_TESTS thetah-sl-test11conv-r0t1-cdr30-rrm-kokkos.cmake) + LIST(APPEND HOMME_TESTS + thetah-sl-test11conv-r0t1-cdr30-rrm-kokkos.cmake + thetah-sl-testconv-3e-kokkos.cmake) IF (HOMMEXX_BFB_TESTING) LIST(APPEND HOMME_ONEOFF_CVF_TESTS thetah-sl-test11conv-r0t1-cdr30-rrm) diff --git a/components/homme/test/reg_test/run_tests/thetah-sl-test11conv-r0t1-cdr30-rrm-kokkos.cmake b/components/homme/test/reg_test/run_tests/thetah-sl-test11conv-r0t1-cdr30-rrm-kokkos.cmake index 0bf617bcdb7e..fb7e41a3d040 100644 --- a/components/homme/test/reg_test/run_tests/thetah-sl-test11conv-r0t1-cdr30-rrm-kokkos.cmake +++ b/components/homme/test/reg_test/run_tests/thetah-sl-test11conv-r0t1-cdr30-rrm-kokkos.cmake @@ -11,4 +11,4 @@ SET(MESH_FILES ${HOMME_ROOT}/test/mesh_files/mountain_10_x2.g) # compare all of these files against baselines: SET(NC_OUTPUT_FILES - dcmip2012_test1_1_conv1.nc) + dcmip2012_test1_3a_conv1.nc) diff --git a/components/homme/test/reg_test/run_tests/thetah-sl-test11conv-r0t1-cdr30-rrm.cmake b/components/homme/test/reg_test/run_tests/thetah-sl-test11conv-r0t1-cdr30-rrm.cmake index 62387e9474b7..688b946ed594 100644 --- a/components/homme/test/reg_test/run_tests/thetah-sl-test11conv-r0t1-cdr30-rrm.cmake +++ b/components/homme/test/reg_test/run_tests/thetah-sl-test11conv-r0t1-cdr30-rrm.cmake @@ -11,4 +11,4 @@ SET(MESH_FILES ${HOMME_ROOT}/test/mesh_files/mountain_10_x2.g) # compare all of these files against baselines: SET(NC_OUTPUT_FILES - dcmip2012_test1_1_conv1.nc) + dcmip2012_test1_3a_conv1.nc) diff --git a/components/homme/test/reg_test/run_tests/thetah-sl-test11conv-r1t2-cdr20.cmake b/components/homme/test/reg_test/run_tests/thetah-sl-test11conv-r1t2-cdr20.cmake index 8dacd43d872b..ca0245833b63 100644 --- a/components/homme/test/reg_test/run_tests/thetah-sl-test11conv-r1t2-cdr20.cmake +++ b/components/homme/test/reg_test/run_tests/thetah-sl-test11conv-r1t2-cdr20.cmake @@ -9,4 +9,4 @@ SET(NAMELIST_FILES ${HOMME_ROOT}/test/reg_test/namelists/thetah-sl-test11conv-r1 # compare all of these files against baselines: SET(NC_OUTPUT_FILES - dcmip2012_test1_1_conv1.nc) + dcmip2012_test1_3a_conv1.nc) diff --git a/components/homme/test/reg_test/run_tests/thetah-sl-testconv-3e-kokkos.cmake b/components/homme/test/reg_test/run_tests/thetah-sl-testconv-3e-kokkos.cmake new file mode 100644 index 000000000000..4d87d00d6552 --- /dev/null +++ b/components/homme/test/reg_test/run_tests/thetah-sl-testconv-3e-kokkos.cmake @@ -0,0 +1,11 @@ +# The name of this test (should be the basename of this file) +SET(TEST_NAME thetah-sl-testconv-3e-kokkos) +# The specifically compiled executable that this test uses +SET(EXEC_NAME theta-l-nlev30-kokkos) + +SET(NUM_CPUS 16) + +SET(NAMELIST_FILES ${HOMME_ROOT}/test/reg_test/namelists/thetah-sl-testconv-3e.nl) + +# compare all of these files against baselines: +SET(NC_OUTPUT_FILES dcmip2012_test1_3e_conv1.nc) diff --git a/components/homme/test/reg_test/run_tests/thetah-sl-testconv-3e.cmake b/components/homme/test/reg_test/run_tests/thetah-sl-testconv-3e.cmake new file mode 100644 index 000000000000..6eaff42f94d2 --- /dev/null +++ b/components/homme/test/reg_test/run_tests/thetah-sl-testconv-3e.cmake @@ -0,0 +1,11 @@ +# The name of this test (should be the basename of this file) +SET(TEST_NAME thetah-sl-testconv-3e) +# The specifically compiled executable that this test uses +SET(EXEC_NAME theta-l-nlev30) + +SET(NUM_CPUS 16) + +SET(NAMELIST_FILES ${HOMME_ROOT}/test/reg_test/namelists/thetah-sl-testconv-3e.nl) + +# compare all of these files against baselines: +SET(NC_OUTPUT_FILES dcmip2012_test1_3e_conv1.nc) diff --git a/components/homme/test/unit_tests/CMakeLists.txt b/components/homme/test/unit_tests/CMakeLists.txt index 2b601ae172cf..b029e73277e7 100644 --- a/components/homme/test/unit_tests/CMakeLists.txt +++ b/components/homme/test/unit_tests/CMakeLists.txt @@ -22,14 +22,17 @@ macro(cxx_unit_test target_name target_f90_srcs target_cxx_srcs include_dirs con cxx_unit_test_add_test(${target_name}_test ${target_name} ${NUM_CPUS}) TARGET_LINK_LIBRARIES(${target_name} timing ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES}) - target_link_libraries(${target_name} Kokkos::kokkos) + TARGET_LINK_LIBRARIES(${target_name} Kokkos::kokkos) IF (HOMME_USE_MKL) - TARGET_COMPILE_OPTIONS (${target_name} PUBLIC -mkl) - TARGET_LINK_LIBRARIES (${target_name} -mkl) + IF (MKL_TYPE STREQUAL "oneMKL") + TARGET_LINK_LIBRARIES(${target_name} -qmkl) + ELSEIF (MKL_TYPE STREQUAL "Intel MKL") + TARGET_LINK_LIBRARIES(${target_name} -mkl) + ENDIF () ENDIF() # Link csm_share lib - target_link_libraries(${target_name} csm_share) + TARGET_LINK_LIBRARIES(${target_name} csm_share) STRING(TOUPPER "${PERFORMANCE_PROFILE}" PERF_PROF_UPPER) IF ("${PERF_PROF_UPPER}" STREQUAL "VTUNE") diff --git a/components/homme/test/unit_tests/tester.cpp b/components/homme/test/unit_tests/tester.cpp index 826257930e8a..0acce65a41db 100644 --- a/components/homme/test/unit_tests/tester.cpp +++ b/components/homme/test/unit_tests/tester.cpp @@ -25,13 +25,13 @@ int main(int argc, char **argv) { Homme::initialize_hommexx_session(); // Filter arguments so catch2 doesn't try to interpret hommexx-specific ones. - hommexx_catch2_argc = argc; + hommexx_catch2_argc = 0; hommexx_catch2_argv = argv; for (int i = 1; i < argc; ++i) { if (std::string(argv[i]) == "hommexx") { - argc = i; - hommexx_catch2_argc -= i + 1; + hommexx_catch2_argc = argc - (i + 1); hommexx_catch2_argv = argv + i + 1; + argc = i; break; } } diff --git a/components/homme/test_execs/CMakeLists.txt b/components/homme/test_execs/CMakeLists.txt index a007a5532b61..12a55d640254 100644 --- a/components/homme/test_execs/CMakeLists.txt +++ b/components/homme/test_execs/CMakeLists.txt @@ -195,9 +195,6 @@ IF(${BUILD_HOMME_PREQX}) IF(${BUILD_HOMME_PREQX_ACC}) ADD_SUBDIRECTORY(baroCam-acc) ENDIF() - IF(${HOMME_ENABLE_COMPOSE}) - ADD_SUBDIRECTORY(stt) - ENDIF() ENDIF() # Add the test exec subdirs for the prim executable @@ -222,6 +219,8 @@ IF(${BUILD_HOMME_THETA}) # ADD_SUBDIRECTORY(theta-l-nlev200-native) # ADD_SUBDIRECTORY(theta-l-nlev256-native) # ADD_SUBDIRECTORY(theta-l-nlev300-native) + # Special test for -DHOMME_WITHOUT_PIOLIBRARY. + ADD_SUBDIRECTORY(stt) ENDIF() IF (${BUILD_HOMME_THETA_KOKKOS}) diff --git a/components/homme/test_execs/preqx_kokkos_ut/CMakeLists.txt b/components/homme/test_execs/preqx_kokkos_ut/CMakeLists.txt index 9dd30f58f63f..3347e7b48944 100644 --- a/components/homme/test_execs/preqx_kokkos_ut/CMakeLists.txt +++ b/components/homme/test_execs/preqx_kokkos_ut/CMakeLists.txt @@ -36,10 +36,14 @@ ADD_LIBRARY(preqx_kokkos_ut_lib TARGET_INCLUDE_DIRECTORIES(preqx_kokkos_ut_lib PUBLIC ${EXEC_INCLUDE_DIRS}) TARGET_INCLUDE_DIRECTORIES(preqx_kokkos_ut_lib PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) TARGET_COMPILE_DEFINITIONS(preqx_kokkos_ut_lib PUBLIC "HAVE_CONFIG_H") -target_link_libraries(preqx_kokkos_ut_lib Kokkos::kokkos) +TARGET_LINK_LIBRARIES(preqx_kokkos_ut_lib Kokkos::kokkos) TARGET_LINK_LIBRARIES(preqx_kokkos_ut_lib timing csm_share ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES}) IF (HOMME_USE_MKL) - TARGET_LINK_LIBRARIES (preqx_kokkos_ut_lib -mkl) + IF (MKL_TYPE STREQUAL "oneMKL") + TARGET_LINK_LIBRARIES(preqx_kokkos_ut_lib -qmkl) + ELSEIF (MKL_TYPE STREQUAL "Intel MKL") + TARGET_LINK_LIBRARIES(preqx_kokkos_ut_lib -mkl) + ENDIF () ENDIF() IF(NOT BUILD_HOMME_WITHOUT_PIOLIBRARY) IF(HOMME_USE_SCORPIO) diff --git a/components/homme/test_execs/stt/CMakeLists.txt b/components/homme/test_execs/stt/CMakeLists.txt index c0268957730e..93b083ca1c5f 100644 --- a/components/homme/test_execs/stt/CMakeLists.txt +++ b/components/homme/test_execs/stt/CMakeLists.txt @@ -1,7 +1,7 @@ -preqx_setup() +thetal_setup() ADD_DEFINITIONS(-DHOMME_WITHOUT_PIOLIBRARY) # Set the variables for this test executable # NP NC PLEV USE_PIO WITH_ENERGY QSIZE_D -createTestExec(stt preqx 4 4 3 FALSE TRUE 5) +createTestExec(stt theta-l 4 4 3 FALSE TRUE 5) diff --git a/components/homme/test_execs/thetal_kokkos_ut/CMakeLists.txt b/components/homme/test_execs/thetal_kokkos_ut/CMakeLists.txt index e8bf5e20bd03..11030b15e876 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/CMakeLists.txt +++ b/components/homme/test_execs/thetal_kokkos_ut/CMakeLists.txt @@ -42,10 +42,14 @@ ADD_LIBRARY(thetal_kokkos_ut_lib TARGET_INCLUDE_DIRECTORIES(thetal_kokkos_ut_lib PUBLIC ${EXEC_INCLUDE_DIRS}) TARGET_INCLUDE_DIRECTORIES(thetal_kokkos_ut_lib PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) TARGET_COMPILE_DEFINITIONS(thetal_kokkos_ut_lib PUBLIC "HAVE_CONFIG_H") -target_link_libraries(thetal_kokkos_ut_lib Kokkos::kokkos) +TARGET_LINK_LIBRARIES(thetal_kokkos_ut_lib Kokkos::kokkos) TARGET_LINK_LIBRARIES(thetal_kokkos_ut_lib timing csm_share ${COMPOSE_LIBRARY_CPP} ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES}) IF (HOMME_USE_MKL) - TARGET_LINK_LIBRARIES (thetal_kokkos_ut_lib -mkl) + IF (MKL_TYPE STREQUAL "oneMKL") + TARGET_LINK_LIBRARIES(thetal_kokkos_ut_lib -qmkl) + ELSEIF (MKL_TYPE STREQUAL "Intel MKL") + TARGET_LINK_LIBRARIES(thetal_kokkos_ut_lib -mkl) + ENDIF () ENDIF() IF(BUILD_HOMME_WITHOUT_PIOLIBRARY) TARGET_COMPILE_DEFINITIONS(thetal_kokkos_ut_lib PUBLIC HOMME_WITHOUT_PIOLIBRARY) diff --git a/components/homme/test_execs/thetal_kokkos_ut/compose_interface.F90 b/components/homme/test_execs/thetal_kokkos_ut/compose_interface.F90 index 4296b6609a08..a34d5411e959 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/compose_interface.F90 +++ b/components/homme/test_execs/thetal_kokkos_ut/compose_interface.F90 @@ -8,7 +8,7 @@ module compose_interface contains subroutine init_compose_f90(ne, hyai, hybi, hyam, hybm, ps0, dvv, mp, qsize_in, hv_q, & - lim, cdr_check, is_sphere) bind(c) + lim, cdr_check, is_sphere, nearest_point, halo, traj_nsubstep) bind(c) use hybvcoord_mod, only: set_layer_locations use thetal_test_interface, only: init_f90 use theta_f2c_mod, only: init_elements_c @@ -16,7 +16,8 @@ subroutine init_compose_f90(ne, hyai, hybi, hyam, hybm, ps0, dvv, mp, qsize_in, use control_mod, only: transport_alg, semi_lagrange_cdr_alg, semi_lagrange_cdr_check, & semi_lagrange_hv_q, limiter_option, nu_q, hypervis_subcycle_q, hypervis_order, & vert_remap_q_alg, qsplit, rsplit, dt_remap_factor, dt_tracer_factor, & - theta_hydrostatic_mode + theta_hydrostatic_mode, semi_lagrange_nearest_point_lev, semi_lagrange_halo, & + semi_lagrange_trajectory_nsubstep use geometry_interface_mod, only: GridVertex use bndry_mod, only: sort_neighbor_buffer_mapping use reduction_mod, only: initreductionbuffer, red_sum, red_min, red_max @@ -25,17 +26,17 @@ subroutine init_compose_f90(ne, hyai, hybi, hyam, hybm, ps0, dvv, mp, qsize_in, use sl_advection, only: sl_init1 real (real_kind), intent(in) :: hyai(nlevp), hybi(nlevp), hyam(nlev), hybm(nlev) - integer (c_int), value, intent(in) :: ne, qsize_in, hv_q, lim + integer (c_int), value, intent(in) :: ne, qsize_in, hv_q, lim, halo, traj_nsubstep real (real_kind), value, intent(in) :: ps0 real (real_kind), intent(out) :: dvv(np,np), mp(np,np) - logical (c_bool), value, intent(in) :: cdr_check, is_sphere + logical (c_bool), value, intent(in) :: cdr_check, is_sphere, nearest_point integer :: ie, edgesz if (.not. is_sphere) print *, "NOT IMPL'ED YET" transport_alg = 12 - semi_lagrange_cdr_alg = 30 + semi_lagrange_cdr_alg = 3 semi_lagrange_cdr_check = cdr_check qsize = qsize_in limiter_option = lim @@ -45,6 +46,10 @@ subroutine init_compose_f90(ne, hyai, hybi, hyam, hybm, ps0, dvv, mp, qsize_in, dt_tracer_factor = -1 dt_remap_factor = -1 theta_hydrostatic_mode = .true. + semi_lagrange_nearest_point_lev = -1 + if (nearest_point) semi_lagrange_nearest_point_lev = 100000 + semi_lagrange_halo = halo + semi_lagrange_trajectory_nsubstep = traj_nsubstep hypervis_order = 2 semi_lagrange_hv_q = hv_q @@ -148,7 +153,7 @@ subroutine run_compose_standalone_test_f90(nmax_out, eval) bind(c) use thread_mod, only: hthreads, vthreads use dimensions_mod, only: nlev, qsize - integer(c_int), intent(out) :: nmax_out + integer(c_int), intent(inout) :: nmax_out real(c_double), intent(out) :: eval((nlev+1)*qsize) type (domain1d_t), pointer :: dom_mt(:) @@ -162,8 +167,12 @@ subroutine run_compose_standalone_test_f90(nmax_out, eval) bind(c) dom_mt(0)%start = 1 dom_mt(0)%end = nelemd transport_alg = 19 - nmax = 7*ne - nmax_out = nmax + if (nmax_out <= 1) then + nmax = 7*ne + nmax_out = nmax + else + nmax = nmax_out + end if statefreq = 2*ne call compose_test(par, hvcoord, dom_mt, elem, buf) do i = 1,size(buf) @@ -190,7 +199,7 @@ subroutine run_trajectory_f90(t0, t1, independent_time_steps, dep, dprecon) bind type (timelevel_t) :: tl type (hybrid_t) :: hybrid real(real_kind) :: dt - integer :: ie, i, j, k, testno, geometry_type + integer :: ie, i, j, k, d, testno, geometry_type logical :: its call timelevel_init_default(tl) @@ -216,9 +225,9 @@ subroutine run_trajectory_f90(t0, t1, independent_time_steps, dep, dprecon) bind do k = 1,nlev do j = 1,np do i = 1,np - dep(1,i,j,k,ie) = dep_points_all(i,j,k,ie)%x - dep(2,i,j,k,ie) = dep_points_all(i,j,k,ie)%y - dep(3,i,j,k,ie) = dep_points_all(i,j,k,ie)%z + do d = 1, 3 + dep(d,i,j,k,ie) = dep_points_all(d,i,j,k,ie) + end do dprecon(i,j,k,ie) = elem(ie)%derived%divdp(i,j,k) end do end do diff --git a/components/homme/test_execs/thetal_kokkos_ut/compose_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/compose_ut.cpp index 0c0d06cff67e..85a300567665 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/compose_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/compose_ut.cpp @@ -37,7 +37,8 @@ extern char** hommexx_catch2_argv; extern "C" { void init_compose_f90(int ne, const Real* hyai, const Real* hybi, const Real* hyam, const Real* hybm, Real ps0, Real* dvv, Real* mp, int qsize, - int hv_q, int limiter_option, bool cdr_check, bool is_sphere); + int hv_q, int limiter_option, bool cdr_check, bool is_sphere, + bool nearest_point, int halo, int traj_nsubstep); void init_geometry_f90(); void cleanup_compose_f90(); void run_compose_standalone_test_f90(int* nmax, Real* eval); @@ -99,8 +100,8 @@ void fill (Random& r, const V& a, } struct Session { - int ne, hv_q; - bool cdr_check, is_sphere; + int ne, hv_q, nmax, halo, traj_nsubstep; + bool cdr_check, is_sphere, run_only_advection_test, nearest_point; HybridVCoord h; Random r; std::shared_ptr e; @@ -113,6 +114,11 @@ struct Session { const auto seed = r.gen_seed(); printf("seed %u\n", seed); + nlev = NUM_PHYSICAL_LEV; + assert(nlev > 0); + np = NP; + assert(np == 4); + assert(QSIZE_D >= 4); parse_command_line(); assert(is_sphere); // planar isn't available in Hxx yet @@ -141,11 +147,12 @@ struct Session { const auto hybi = cmvdc(h.hybrid_bi); const auto hyam = cmvdc(h.hybrid_am); const auto hybm = cmvdc(h.hybrid_bm); + auto& ref_FE = c.create(); std::vector dvv(NP*NP), mp(NP*NP); init_compose_f90(ne, hyai.data(), hybi.data(), &hyam(0)[0], &hybm(0)[0], h.ps0, dvv.data(), mp.data(), qsize, hv_q, p.limiter_option, cdr_check, - is_sphere); + is_sphere, nearest_point, halo, traj_nsubstep); ref_FE.init_mass(mp.data()); ref_FE.init_deriv(dvv.data()); @@ -168,11 +175,6 @@ struct Session { ct.init_buffers(fbm); ct.init_boundary_exchanges(); - nlev = NUM_PHYSICAL_LEV; - assert(nlev > 0); - np = NP; - assert(np == 4); - c.create(); } @@ -203,37 +205,87 @@ struct Session { // compose_ut hommexx -ne NE -qsize QSIZE -hvq HV_Q -cdrcheck void parse_command_line () { - const bool am_root = get_comm().root(); ne = 2; qsize = QSIZE_D; hv_q = 1; cdr_check = false; is_sphere = true; + run_only_advection_test = false; + nmax = -1; + halo = 2; + traj_nsubstep = 0; + nearest_point = true; + + const bool am_root = get_comm().root(); bool ok = true; int i; for (i = 0; i < hommexx_catch2_argc; ++i) { const std::string tok(hommexx_catch2_argv[i]); if (tok == "-ne") { - if (i+1 == hommexx_catch2_argc) { ok = false; break; } + if (i+1 == hommexx_catch2_argc) ok = false; ne = std::atoi(hommexx_catch2_argv[++i]); + if (ne < 2) { + printf("ne must be >= 2\n"); + ok = false; + } } else if (tok == "-qsize") { - if (i+1 == hommexx_catch2_argc) { ok = false; break; } + if (i+1 == hommexx_catch2_argc) ok = false; qsize = std::atoi(hommexx_catch2_argv[++i]); + if (qsize > QSIZE_D || qsize < 1) { + printf("qsize must be >= 1 and <= QSIZE_D\n"); + ok = false; + } } else if (tok == "-hvq") { - if (i+1 == hommexx_catch2_argc) { ok = false; break; } + if (i+1 == hommexx_catch2_argc) ok = false; hv_q = std::atoi(hommexx_catch2_argv[++i]); } else if (tok == "-cdrcheck") { cdr_check = true; } else if (tok == "-planar") { is_sphere = false; + } else if (tok == "-convergence") { + // When running this as a convergence-test driver, don't run any tests + // except the prescribed-flow one. + run_only_advection_test = true; + } else if (tok == "-nmax") { + if (i+1 == hommexx_catch2_argc) ok = false; + nmax = std::atoi(hommexx_catch2_argv[++i]); + if (nmax < 1) { + printf("nmax must be >= 1\n"); + ok = false; + } + } else if (tok == "-halo") { + if (i+1 == hommexx_catch2_argc) ok = false; + halo = std::atoi(hommexx_catch2_argv[++i]); + if (halo < 1) { + printf("halo must be >= 1"); + ok = false; + } + } else if (tok == "-trajnsubstep") { + if (i+1 == hommexx_catch2_argc) ok = false; + traj_nsubstep = std::atoi(hommexx_catch2_argv[++i]); + if (traj_nsubstep < 0) { + printf("traj_nsubstep must be >= 0\n"); + ok = false; + } + } else if (tok == "-nonearest") { + nearest_point = false; + } else { + printf("unrecognized token %s\n", tok.c_str()); + ok = false; } + if ( ! ok) break; } - ne = std::max(2, std::min(128, ne)); + + ne = std::max(2, ne); qsize = std::max(1, std::min(QSIZE_D, qsize)); hv_q = std::max(0, std::min(qsize, hv_q)); - if ( ! ok && am_root) + + if ( ! ok && am_root) { printf("compose_ut> Failed to parse command line, starting with: %s\n", hommexx_catch2_argv[i]); + Homme::Errors::runtime_abort("compose_ut invalid command line"); + } + if (am_root) { const int bfb = #ifdef HOMMEXX_BFB_TESTING @@ -241,8 +293,10 @@ struct Session { #else 0; #endif - printf("compose_ut> bfb %d ne %d qsize %d hv_q %d cdr_check %d\n", - bfb, ne, qsize, hv_q, cdr_check ? 1 : 0); + printf("compose_ut> sphere %d bfb %d ne %d qsize %d hv_q %d cdr_check %d " + "halo %d traj_nsubstep %d nearest %d\n", + int(is_sphere), bfb, ne, qsize, hv_q, cdr_check ? 1 : 0, halo, + traj_nsubstep, int(nearest_point)); } } }; @@ -337,10 +391,19 @@ TEST_CASE ("compose_transport_testing") { static constexpr Real tol = std::numeric_limits::epsilon(); auto& s = Session::singleton(); try { + do { // breakable + + if (s.run_only_advection_test) { + int nmax = s.nmax; + std::vector eval_f((s.nlev+1)*s.qsize); + run_compose_standalone_test_f90(&nmax, eval_f.data()); + break; + } // unit tests REQUIRE(compose::test::slmm_unittest() == 0); REQUIRE(compose::test::cedr_unittest() == 0); + REQUIRE(compose::test::interpolate_unittest() == 0); REQUIRE(compose::test::cedr_unittest(s.get_comm().mpi_comm()) == 0); auto& ct = Context::singleton().get(); @@ -349,33 +412,35 @@ TEST_CASE ("compose_transport_testing") { REQUIRE(fails.empty()); // trajectory BFB - for (const bool independent_time_steps : {false, true}) { - printf("independent_time_steps %d\n", independent_time_steps); - const Real twelve_days = 3600 * 24 * 12; - const Real t0 = 0.13*twelve_days; - const Real t1 = independent_time_steps ? t0 + 1800 : 0.22*twelve_days; - CA5d depf("depf", s.nelemd, s.nlev, s.np, s.np, 3); - CA4d dpreconf("dpreconf", s.nelemd, s.nlev, s.np, s.np); - run_trajectory_f90(t0, t1, independent_time_steps, depf.data(), - dpreconf.data()); - const auto depc = ct.test_trajectory(t0, t1, independent_time_steps); - REQUIRE(depc.extent_int(0) == s.nelemd); - REQUIRE(depc.extent_int(2) == s.np); - REQUIRE(depc.extent_int(4) == 3); - if (independent_time_steps) { - const auto dpreconc = cmvdc(RNlev(pack2real(s.e->m_derived.m_divdp), s.nelemd)); + if (s.traj_nsubstep == 0) { + for (const bool independent_time_steps : {false, true}) { + printf("independent_time_steps %d\n", independent_time_steps); + const Real twelve_days = 3600 * 24 * 12; + const Real t0 = 0.13*twelve_days; + const Real t1 = independent_time_steps ? t0 + 1800 : 0.22*twelve_days; + CA5d depf("depf", s.nelemd, s.nlev, s.np, s.np, 3); + CA4d dpreconf("dpreconf", s.nelemd, s.nlev, s.np, s.np); + run_trajectory_f90(t0, t1, independent_time_steps, depf.data(), + dpreconf.data()); + const auto depc = ct.test_trajectory(t0, t1, independent_time_steps); + REQUIRE(depc.extent_int(0) == s.nelemd); + REQUIRE(depc.extent_int(2) == s.np); + REQUIRE(depc.extent_int(4) == 3); + if (independent_time_steps) { + const auto dpreconc = cmvdc(RNlev(pack2real(s.e->m_derived.m_divdp), s.nelemd)); + for (int ie = 0; ie < s.nelemd; ++ie) + for (int lev = 0; lev < s.nlev; ++lev) + for (int i = 0; i < s.np; ++i) + for (int j = 0; j < s.np; ++j) + REQUIRE(equal(dpreconf(ie,lev,i,j), dpreconc(ie,i,j,lev), 100*tol)); + } for (int ie = 0; ie < s.nelemd; ++ie) for (int lev = 0; lev < s.nlev; ++lev) for (int i = 0; i < s.np; ++i) for (int j = 0; j < s.np; ++j) - REQUIRE(equal(dpreconf(ie,lev,i,j), dpreconc(ie,i,j,lev), 10*tol)); + for (int d = 0; d < 3; ++d) + REQUIRE(equal(depf(ie,lev,i,j,d), depc(ie,lev,i,j,d), 100*tol)); } - for (int ie = 0; ie < s.nelemd; ++ie) - for (int lev = 0; lev < s.nlev; ++lev) - for (int i = 0; i < s.np; ++i) - for (int j = 0; j < s.np; ++j) - for (int d = 0; d < 3; ++d) - REQUIRE(equal(depf(ie,lev,i,j,d), depc(ie,lev,i,j,d), 10*tol)); } { // q vertical remap @@ -386,7 +451,7 @@ TEST_CASE ("compose_transport_testing") { } { // 2D SL BFB - int nmax; + int nmax = s.nmax; std::vector eval_f((s.nlev+1)*s.qsize), eval_c(eval_f.size()); run_compose_standalone_test_f90(&nmax, eval_f.data()); for (const bool bfb : {false, true}) { @@ -397,9 +462,10 @@ TEST_CASE ("compose_transport_testing") { if (s.get_comm().root()) { const Real f = bfb ? 0 : 1; const int n = s.nlev*s.qsize; - // When not a BFB build, still expect l2 error to be the same to a few digits. - for (int i = 0; i < n; ++i) REQUIRE(almost_equal(eval_f[i], eval_c[i], f*1e-3)); - // Mass conservation error should be within a factor of 10 of each other. + // When not a BFB build, still expect l2 error to be the same to several digits. + for (int i = 0; i < n; ++i) REQUIRE(almost_equal(eval_f[i], eval_c[i], f*1e5*tol)); + // Mass conservation error should be within a factor of 10 of each + // other. for (int i = n; i < n + s.qsize; ++i) REQUIRE(almost_equal(eval_f[i], eval_c[i], f*10)); // And mass conservation itself should be small. for (int i = n; i < n + s.qsize; ++i) REQUIRE(std::abs(eval_f[i]) <= 20*tol); @@ -409,6 +475,7 @@ TEST_CASE ("compose_transport_testing") { } } + } while (false); // do } catch (...) {} Session::delete_singleton(); } diff --git a/components/mosart/src/riverroute/MOSART_physics_mod.F90 b/components/mosart/src/riverroute/MOSART_physics_mod.F90 index 912014904e93..8d3843abad52 100644 --- a/components/mosart/src/riverroute/MOSART_physics_mod.F90 +++ b/components/mosart/src/riverroute/MOSART_physics_mod.F90 @@ -680,10 +680,12 @@ subroutine Euler end do ! DLevelH2R ! subcycling within MOSART ends -! check for negative channel storage - if (negchan < -1.e-10) then - write(iulog,*) 'Warning: Negative channel storage found! ',negchan -! call shr_sys_abort('mosart: negative channel storage') + ! check for negative channel storage + if (negchan < -1.e-10 .and. negchan >= -1.e-8) then + write(iulog,*) 'Warning: Small negative channel storage found! ',negchan + elseif(negchan < -1.e-8) then + write(iulog,*) 'Error: Negative channel storage found! ',negchan + call shr_sys_abort('mosart: negative channel storage') endif TRunoff%flow = TRunoff%flow / Tctl%DLevelH2R TRunoff%erowm_regi(:,nt_nmud:nt_nsan) = TRunoff%erowm_regi(:,nt_nmud:nt_nsan) / Tctl%DLevelH2R @@ -876,11 +878,7 @@ subroutine Routing_KW(iunit, nt, theDeltaT) TRunoff%erout(iunit,nt) = -TRunoff%vr(iunit,nt) * TRunoff%mr(iunit,nt) if(-TRunoff%erout(iunit,nt) > TINYVALUE .and. TRunoff%wr(iunit,nt) + & (TRunoff%erlateral(iunit,nt) + TRunoff%erin(iunit,nt) + TRunoff%erout(iunit,nt)) * theDeltaT < TINYVALUE) then - if (sediflag) then - TRunoff%erout(iunit,nt) = -(TRunoff%erlateral(iunit,nt) + TRunoff%erin(iunit,nt) + TRunoff%wr(iunit,nt)*MaxStorageDepleted/ theDeltaT) - else - TRunoff%erout(iunit,nt) = -(TRunoff%erlateral(iunit,nt) + TRunoff%erin(iunit,nt) + TRunoff%wr(iunit,nt)/ theDeltaT) - end if + TRunoff%erout(iunit,nt) = -(TRunoff%erlateral(iunit,nt) + TRunoff%erin(iunit,nt) + TRunoff%wr(iunit,nt)*MaxStorageDepleted/ theDeltaT) if(TRunoff%mr(iunit,nt) > 0._r8) then TRunoff%vr(iunit,nt) = -TRunoff%erout(iunit,nt) / TRunoff%mr(iunit,nt) end if @@ -918,16 +916,6 @@ subroutine Routing_KW(iunit, nt, theDeltaT) TRunoff%dwr(iunit,nt) = TRunoff%erlateral(iunit,nt) + TRunoff%erin(iunit,nt) + TRunoff%erout(iunit,nt) + temp_gwl - !if(TRunoff%wr(iunit,nt) < TINYVALUE .and. abs(TRunoff%erout(iunit,nt))> TINYVALUE) then - ! write(unit=1111,fmt="(i10, 4(e20.11))") iunit, TRunoff%wr(iunit,nt), TRunoff%erout(iunit,nt), TRunoff%erlateral(iunit,nt) + TRunoff%erin(iunit,nt), TRunoff%dwr(iunit,nt) - ! write(unit=1112,fmt="(2(i10), 4(e20.11))") iunit, TUnit%mask(iunit), TRunoff%vr(iunit,nt), TUnit%rlen(iunit), TUnit%rwidth(iunit), TUnit%areaTotal2(iunit)/TUnit%rwidth(iunit)/TUnit%rlen(iunit) - !end if - -! if(iunit==490 .and. nt==1) then -! write(unit=1111,fmt="(3(e20.11), 5(f12.4))") TUnit%areaTotal2(iunit),TUnit%areaTotal(iunit),TUnit%area(iunit),TUnit%rdepth(iunit), TUnit%rwidth(iunit), TUnit%rslp(iunit), TUnit%nr(iunit), TUnit%nt(iunit) -! write(unit=1112,fmt="(6(e20.11))") TRunoff%wr(iunit,nt), TRunoff%dwr(iunit,nt), TRunoff%erlateral(iunit,nt), TRunoff%erin(iunit,nt), TRunoff%erout(iunit,nt), temp_gwl -! end if - ! check for stability ! if(TRunoff%vr(iunit,nt) < -TINYVALUE .or. TRunoff%vr(iunit,nt) > 30) then ! write(iulog,*) "Numerical error inRouting_KW, ", iunit,nt,TRunoff%vr(iunit,nt) diff --git a/components/mpas-albany-landice/bld/build-namelist b/components/mpas-albany-landice/bld/build-namelist index 15272f7a2dab..1f230c7543ec 100755 --- a/components/mpas-albany-landice/bld/build-namelist +++ b/components/mpas-albany-landice/bld/build-namelist @@ -559,6 +559,10 @@ add_default($nl, 'config_alpha_subglacial_discharge'); add_default($nl, 'config_subglacial_discharge_coefficient'); add_default($nl, 'config_subglacial_discharge_intercept'); add_default($nl, 'config_uniform_face_melt_rate'); +add_default($nl, 'config_ocean_data_extrapolation'); +add_default($nl, 'config_ocean_data_extrap_ncells_extra'); +add_default($nl, 'config_invalid_value_TF'); +add_default($nl, 'config_weight_value_cell'); ####################################### # Namelist group: physical_parameters # @@ -620,6 +624,7 @@ add_default($nl, 'config_pio_stride'); add_default($nl, 'config_year_digits'); add_default($nl, 'config_output_external_velocity_solver_data'); add_default($nl, 'config_write_albany_ascii_mesh'); +add_default($nl, 'config_create_all_logs_in_e3sm'); ################################# # Namelist group: decomposition # diff --git a/components/mpas-albany-landice/bld/build-namelist-section b/components/mpas-albany-landice/bld/build-namelist-section index 4b5e05ec3cd5..14c0732bc2cf 100644 --- a/components/mpas-albany-landice/bld/build-namelist-section +++ b/components/mpas-albany-landice/bld/build-namelist-section @@ -121,6 +121,10 @@ add_default($nl, 'config_alpha_subglacial_discharge'); add_default($nl, 'config_subglacial_discharge_coefficient'); add_default($nl, 'config_subglacial_discharge_intercept'); add_default($nl, 'config_uniform_face_melt_rate'); +add_default($nl, 'config_ocean_data_extrapolation'); +add_default($nl, 'config_ocean_data_extrap_ncells_extra'); +add_default($nl, 'config_invalid_value_TF'); +add_default($nl, 'config_weight_value_cell'); ####################################### # Namelist group: physical_parameters # @@ -182,6 +186,7 @@ add_default($nl, 'config_pio_stride'); add_default($nl, 'config_year_digits'); add_default($nl, 'config_output_external_velocity_solver_data'); add_default($nl, 'config_write_albany_ascii_mesh'); +add_default($nl, 'config_create_all_logs_in_e3sm'); ################################# # Namelist group: decomposition # diff --git a/components/mpas-albany-landice/bld/namelist_files/namelist_defaults_mali.xml b/components/mpas-albany-landice/bld/namelist_files/namelist_defaults_mali.xml index 0450eb44f116..b437bcdfc830 100644 --- a/components/mpas-albany-landice/bld/namelist_files/namelist_defaults_mali.xml +++ b/components/mpas-albany-landice/bld/namelist_files/namelist_defaults_mali.xml @@ -18,7 +18,7 @@ 'fo' -'none' +'fo' 3 0.25 .false. @@ -98,7 +98,7 @@ 1.0 0.0 0.25 -'none' +'ismip6' .false. 1.18 0.0 @@ -106,6 +106,10 @@ 3.0e-4 0.15 0.0 +.false. +10 +1.0e36 +0.9 910.0 @@ -150,6 +154,7 @@ 4 .false. .false. +.false. 3 diff --git a/components/mpas-albany-landice/bld/namelist_files/namelist_definition_mali.xml b/components/mpas-albany-landice/bld/namelist_files/namelist_definition_mali.xml index e16e7042a2cb..3ea4ef5e4786 100644 --- a/components/mpas-albany-landice/bld/namelist_files/namelist_definition_mali.xml +++ b/components/mpas-albany-landice/bld/namelist_files/namelist_definition_mali.xml @@ -795,6 +795,38 @@ Valid values: any non-negative value Default: Defined in namelist_defaults.xml + +If true, extrapolate ocean data (temperature, salinity, thermal forcing) from external source into underneath the ice draft. + +Valid values: .true. or .false. +Default: Defined in namelist_defaults.xml + + + +number of extra cells for over-extrapolation into grounded ice + +Valid values: any non-negative value +Default: Defined in namelist_defaults.xml + + + +value assigned to indicate invalid thermal forcing value when config_ocean_data_extrapolation is set to true + +Valid values: Any real value +Default: Defined in namelist_defaults.xml + + + +weight used to smooth horizontal extrapolation of ocean data field. Value close to 1 implies more weight of the current cell value versus the averaged value of the neighbouring cells around the cell + +Valid values: any real value between 0 and 1 +Default: Defined in namelist_defaults.xml + + @@ -1087,6 +1119,14 @@ Valid values: .true. or .false. Default: Defined in namelist_defaults.xml + +Logical flag determining if log files will be created for each processor in an E3SM configuration. If .true., the model initializes and writes to one files per processor. + +Valid values: .true. or .false. +Default: Defined in namelist_defaults.xml + + diff --git a/components/mpas-albany-landice/cime_config/buildnml b/components/mpas-albany-landice/cime_config/buildnml index 9489b6dfa8fd..d0fd5551e813 100755 --- a/components/mpas-albany-landice/cime_config/buildnml +++ b/components/mpas-albany-landice/cime_config/buildnml @@ -88,7 +88,7 @@ def buildnml(case, caseroot, compname): decomp_date += '051920' decomp_prefix += 'mpasli.graph.info.' elif glc_grid == 'mpas.gis1to10kmR2': - grid_date += '20230202' + grid_date += '20240513' grid_prefix += 'gis_1to10km_r02' decomp_date += '020223' decomp_prefix += 'mpasli.graph.info.' @@ -247,6 +247,9 @@ def buildnml(case, caseroot, compname): lines.append(' ') lines.append(' ') lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') @@ -280,7 +283,9 @@ def buildnml(case, caseroot, compname): lines.append(' ') lines.append(' ') lines.append(' ') - lines.append(' ') + if mali_use_albany: + lines.append(' ') + lines.append('') lines.append('') lines.append(' + + @@ -896,6 +919,8 @@ + + @@ -1382,6 +1407,9 @@ is the value of that variable from the *previous* time level! + @@ -1433,6 +1461,14 @@ is the value of that variable from the *previous* time level! description="temporary copy of cellMask" persistence="scratch" /> + + @@ -1643,6 +1679,34 @@ is the value of that variable from the *previous* time level! /> + + + + + + + + + + + + @@ -1727,7 +1791,6 @@ is the value of that variable from the *previous* time level! description="grow mask for flood fill" persistence="scratch" /> - diff --git a/components/mpas-albany-landice/src/Registry_subglacial_hydro.xml b/components/mpas-albany-landice/src/Registry_subglacial_hydro.xml index 8b3f07d4750f..ac756b6d33f9 100644 --- a/components/mpas-albany-landice/src/Registry_subglacial_hydro.xml +++ b/components/mpas-albany-landice/src/Registry_subglacial_hydro.xml @@ -178,8 +178,10 @@ + + description="Effective pressure used in basal friction calculation. If subglacial hydrology model is active, this will be effectivePressureSGH averaged over the subglacial hydrology model timestepping subcycles. If subglacial hydrology model is inactive, this will come from a file or a parameterization."/> thermalCellMaskField % array @@ -366,12 +376,14 @@ subroutine li_advection_thickness_tracers(& ! given the old thickness, compute the thickness in each layer call li_calculate_layerThickness(meshPool, thickness, layerThickness) - ! update masks - call li_calculate_mask(meshPool, velocityPool, geometryPool, err_tmp) - err = ior(err, err_tmp) - - ! save old copycellMask for determining cells changing from grounded to floating and vice versa + ! Save copies of masks because we need to preserve mask + ! states prior to advection for accurate time integration. + ! A mask update is necessary to calculate grounding line flux, + ! after which we will reset the masks to their previous states. cellMaskTemporaryField % array(:) = cellMask(:) + edgeMaskTemporaryField % array(:) = edgeMask(:) + vertexMaskTemporaryField % array(:) = vertexMask(:) + layerThicknessEdgeFlux(:,:) = 0.0_RKIND !----------------------------------------------------------------- @@ -540,10 +552,10 @@ subroutine li_advection_thickness_tracers(& dynamicThickening = (sum(layerThickness, 1) - thickness) / dt * scyr ! units of m/yr - ! Update the thickness and cellMask before applying the mass balance. - ! The update is needed because the SMB and BMB depend on whether ice is present. + ! Update the thickness before applying the mass balance, but + ! do not update masks because mass balance acts on geometry + ! before advection took place. thickness = sum(layerThickness, 1) - call li_calculate_mask(meshPool, velocityPool, geometryPool, err_tmp) @@ -627,6 +639,9 @@ subroutine li_advection_thickness_tracers(& enddo endif + ! We need an updated set of masks to calculate fluxAcrossGroundingLine, + ! but we will reset this to the previous state below for accuracy of the + ! time integration scheme. call li_calculate_mask(meshPool, velocityPool, geometryPool, err_tmp) err = ior(err, err_tmp) @@ -667,12 +682,17 @@ subroutine li_advection_thickness_tracers(& endif enddo ! edges + ! Reset masks to state before advection and mass balance for + ! accuracy of time integration scheme. + cellMask(:) = cellMaskTemporaryField % array(:) + edgeMask(:) = edgeMaskTemporaryField % array(:) + vertexMask(:) = vertexMaskTemporaryField % array(:) ! Remap tracers to the standard vertical sigma coordinate ! Note: If tracers are not being advected, then this subroutine simply restores the ! layer thickness to sigma coordinate values. - call vertical_remap(thickness, cellMask, meshPool, layerThickness, advectedTracers, err_tmp) + call vertical_remap(thickness, meshPool, layerThickness, advectedTracers, err_tmp) err = ior(err, err_tmp) if (config_print_thickness_advection_info) then @@ -717,13 +737,7 @@ subroutine li_advection_thickness_tracers(& ! Deallocate arrays for fct if ( (trim(config_thickness_advection) .eq. 'fct') .or. & (trim(config_tracer_advection) .eq. 'fct') ) then - deallocate( nAdvCellsForEdge, & - advCellsForEdge, & - advCoefs, & - advCoefs3rd, & - advMaskHighOrder, & - advMask2ndOrder, & - tend) + deallocate(tend) endif ! clean up @@ -733,6 +747,8 @@ subroutine li_advection_thickness_tracers(& call mpas_deallocate_scratch_field(basalTracersField, .true.) call mpas_deallocate_scratch_field(surfaceTracersField, .true.) call mpas_deallocate_scratch_field(cellMaskTemporaryField, .true.) + call mpas_deallocate_scratch_field(edgeMaskTemporaryField, .true.) + call mpas_deallocate_scratch_field(vertexMaskTemporaryField, .true.) call mpas_deallocate_scratch_field(thermalCellMaskField, .true.) ! === error check @@ -1093,7 +1109,6 @@ subroutine tracer_setup(& nTracers) use li_thermal, only: li_temperature_to_enthalpy_kelvin - use li_tracer_advection_fct_shared use li_tracer_advection_fct !----------------------------------------------------------------- ! @@ -1168,11 +1183,9 @@ subroutine tracer_setup(& real (kind=RKIND), pointer :: & config_ice_density ! ice density - integer :: iCell, iTracer, k, err, err1, err2 + integer :: iCell, iTracer, k, err err = 0 - err1 = 0 - err2 = 0 ! get dimensions call mpas_pool_get_dimension(meshPool, 'nCells', nCells) @@ -1301,14 +1314,12 @@ subroutine tracer_setup(& ! May need to increase maxTracers in the Registry. if ( (trim(config_tracer_advection) == 'fct') .or. & (trim(config_thickness_advection) == 'fct') ) then - call li_tracer_advection_fct_shared_init(geometryPool, err1) - call li_tracer_advection_fct_init(err2) + call li_tracer_advection_fct_init(err) - if (err1 /= 0 .or. err2 /= 0) then - err = 1 + if (err /= 0) then call mpas_log_write( & 'Error encountered during fct tracer advection init', & - MPAS_LOG_ERR, masterOnly=.true.) + MPAS_LOG_ERR) endif endif @@ -1950,7 +1961,7 @@ end subroutine li_layer_normal_velocity !> OpenMP over either blocks or cells. ! !----------------------------------------------------------------------- - subroutine vertical_remap(thickness, cellMask, meshPool, layerThickness, tracers, err) + subroutine vertical_remap(thickness, meshPool, layerThickness, tracers, err) !----------------------------------------------------------------- ! @@ -1964,9 +1975,6 @@ subroutine vertical_remap(thickness, cellMask, meshPool, layerThickness, tracers real(kind=RKIND), dimension(:), intent(in) :: & thickness !< Input: ice thickness - integer, dimension(:), intent(in) :: & - cellMask !< Input: mask for cells (needed for determining presence/absence of ice) - !----------------------------------------------------------------- ! ! input/output variables diff --git a/components/mpas-albany-landice/src/mode_forward/mpas_li_calving.F b/components/mpas-albany-landice/src/mode_forward/mpas_li_calving.F index a460c3224a0e..1637c7d697df 100644 --- a/components/mpas-albany-landice/src/mode_forward/mpas_li_calving.F +++ b/components/mpas-albany-landice/src/mode_forward/mpas_li_calving.F @@ -3828,9 +3828,8 @@ subroutine li_finalize_damage_after_advection(domain, err) call mpas_pool_get_array(geometryPool, 'damageNye', damageNye) call mpas_pool_get_array(velocityPool, 'stiffnessFactor', stiffnessFactor) - ! make sure masks are up to date. May not be necessary, but safer to do anyway. - call li_calculate_mask(meshPool, velocityPool, geometryPool, err_tmp) - err = ior(err, err_tmp) + ! Note: In order to preserve accuracy of time integration, + ! we do not update masks before finalizing damage. if (config_preserve_damage) then do iCell = 1, nCells diff --git a/components/mpas-albany-landice/src/mode_forward/mpas_li_core_interface.F b/components/mpas-albany-landice/src/mode_forward/mpas_li_core_interface.F index 8a278675da36..f9f0185499c1 100644 --- a/components/mpas-albany-landice/src/mode_forward/mpas_li_core_interface.F +++ b/components/mpas-albany-landice/src/mode_forward/mpas_li_core_interface.F @@ -108,6 +108,7 @@ function li_setup_packages(configPool, packagePool, iocontext) result(ierr) logical, pointer :: config_SGH logical, pointer :: config_adaptive_timestep_include_DCFL logical, pointer :: config_write_albany_ascii_mesh + logical, pointer :: config_ocean_data_extrapolation logical, pointer :: higherOrderVelocityActive logical, pointer :: SIAvelocityActive @@ -116,6 +117,7 @@ function li_setup_packages(configPool, packagePool, iocontext) result(ierr) logical, pointer :: ismip6ShelfMeltActive logical, pointer :: ismip6GroundedFaceMeltActive logical, pointer :: thermalActive + logical, pointer :: extrapOceanDataActive ierr = 0 @@ -126,6 +128,7 @@ function li_setup_packages(configPool, packagePool, iocontext) result(ierr) call mpas_pool_get_config(configPool, 'config_basal_mass_bal_float', config_basal_mass_bal_float) call mpas_pool_get_config(configPool, 'config_front_mass_bal_grounded', config_front_mass_bal_grounded) call mpas_pool_get_config(configPool, 'config_use_3d_thermal_forcing_for_face_melt', config_use_3d_thermal_forcing_for_face_melt) + call mpas_pool_get_config(configPool, 'config_ocean_data_extrapolation', config_ocean_data_extrapolation) call mpas_pool_get_config(configPool, 'config_thermal_solver', config_thermal_solver) call mpas_pool_get_package(packagePool, 'SIAvelocityActive', SIAvelocityActive) @@ -135,6 +138,7 @@ function li_setup_packages(configPool, packagePool, iocontext) result(ierr) call mpas_pool_get_package(packagePool, 'ismip6ShelfMeltActive', ismip6ShelfMeltActive) call mpas_pool_get_package(packagePool, 'ismip6GroundedFaceMeltActive', ismip6GroundedFaceMeltActive) call mpas_pool_get_package(packagePool, 'thermalActive', thermalActive) + call mpas_pool_get_package(packagePool, 'extrapOceanDataActive', extrapOceanDataActive) if (trim(config_velocity_solver) == 'sia') then SIAvelocityActive = .true. @@ -157,13 +161,21 @@ function li_setup_packages(configPool, packagePool, iocontext) result(ierr) "'config_write_albany_ascii_mesh' is set to .true.") endif + if (config_ocean_data_extrapolation) then + extrapOceanDataActive = .true. + call mpas_log_write("The 'extrapOceanDataActive' package and assocated variables have been enabled because " // & + "'config_ocean_data_extrapolation' is set to .true.") + endif + if ( (trim(config_basal_mass_bal_float) == 'ismip6') .or. & ((trim(config_front_mass_bal_grounded) == 'ismip6') .and. & - (config_use_3d_thermal_forcing_for_face_melt)) ) then + (config_use_3d_thermal_forcing_for_face_melt)) .or. & + (config_ocean_data_extrapolation) ) then ismip6ShelfMeltActive = .true. call mpas_log_write("The 'ismip6Melt' package and assocated variables have been enabled because " // & "'config_basal_mass_bal_float' is set to 'ismip6' or 'config_front_mass_bal_grounded' is set to 'ismip6' // & - and 'config_use_3d_thermal_forcing_for_face_melt' is set to .true.") + and 'config_use_3d_thermal_forcing_for_face_melt' is set to .true. " // & + "or 'config_ocean_data_extrapolation' is set to .true.") endif if ((trim(config_front_mass_bal_grounded) == 'ismip6') .or. (trim(config_calving) == 'ismip6_retreat')) then diff --git a/components/mpas-albany-landice/src/mode_forward/mpas_li_iceshelf_melt.F b/components/mpas-albany-landice/src/mode_forward/mpas_li_iceshelf_melt.F index 5a92ecba377d..daa1d5cca87d 100644 --- a/components/mpas-albany-landice/src/mode_forward/mpas_li_iceshelf_melt.F +++ b/components/mpas-albany-landice/src/mode_forward/mpas_li_iceshelf_melt.F @@ -1162,6 +1162,8 @@ end subroutine calc_iceshelf_draft_info subroutine iceshelf_melt_ismip6(domain, err) + use li_constants, only: oceanFreezingTempDepthDependence + !----------------------------------------------------------------- ! input/output variables !----------------------------------------------------------------- @@ -1182,6 +1184,7 @@ subroutine iceshelf_melt_ismip6(domain, err) integer :: ksup, kk, kinf integer, pointer :: nCellsSolve real(kind=RKIND), pointer :: rhoi, rhosw + real (kind=RKIND), pointer :: invalid_value_TF real(kind=RKIND) :: cste type (block_type), pointer :: block type (mpas_pool_type), pointer :: geometryPool, meshPool @@ -1206,6 +1209,7 @@ subroutine iceshelf_melt_ismip6(domain, err) call mpas_pool_get_config(liConfigs, 'config_ice_density', rhoi) call mpas_pool_get_config(liConfigs, 'config_ocean_density', rhosw) + call mpas_pool_get_config(liConfigs, 'config_invalid_value_TF', invalid_value_TF) cste = (rhosw*cp_seawater/(rhoi*latent_heat_ice))**2 ! in K^(-2) allocate(mean_TF(maxNumBasins)) @@ -1247,28 +1251,65 @@ subroutine iceshelf_melt_ismip6(domain, err) mean_TF(:) = 0.d0 IS_area(:) = 0.d0 + ! Check zOcean for valid values + if (minval(zOcean) >= 0.0_RKIND) then + call mpas_log_write("Invalid value for zOcean. It should have negative values but min value >= 0.0 was found", & + MPAS_LOG_ERR) + err = ior(err, 1) + endif + if (maxval(zOcean) > 0.0_RKIND) then + call mpas_log_write("Invalid value for zOcean. It should have non-positive values but max value greater than 0.0 was found", & + MPAS_LOG_ERR) + err = ior(err, 1) + endif + do iCell = 1, nCellsSolve if ( li_mask_is_floating_ice(cellMask(iCell)) ) then ! 1 - Linear interpolation of the thermal forcing on the ice draft depth : - ksup=1 - do kk=2,nOceanLayers-1 + ksup=0 + do kk=1,nOceanLayers if ( zOcean(kk) >= lowerSurface(iCell) ) ksup = kk enddo kinf = ksup + 1 - if ((zOcean(ksup)-zOcean(kinf)) == 0) then - call mpas_log_write("iceshelf_melt_ismip6: Invalid value for zOcean. " // & - "ksup=$i kinf=$i zOcean(ksup)=$r zOcean(kinf)=$r indexToCellID=$i lowerSurface=$r", MPAS_LOG_ERR, & - intArgs=(/ksup, kinf, indexToCellID(iCell)/), & - realArgs=(/zOcean(ksup), zOcean(kinf), lowerSurface(iCell) /) ) - err = ior(err, 1) - endif + !call mpas_log_write("kinf=$i, zOcean(kinf)=$r, TFocean=$r",realArgs=(/zOcean(kinf),TFocean(kinf,iCell)/), & ! intArgs=(/kinf/)) !call mpas_log_write("ksup=$i, zOcean(ksup)=$r, TFocean=$r",realArgs=(/zOcean(ksup),TFocean(ksup,iCell)/), & ! intArgs=(/ksup/)) - TFdraft(iCell) = ( (zOcean(ksup)-lowerSurface(iCell)) * TFocean(kinf, iCell) & - + (lowerSurface(iCell)-zOcean(kinf)) * TFocean(ksup, iCell) ) / (zOcean(ksup)-zOcean(kinf)) + + ! check if any invalid TFocean value is used for calculating TF at the draft + if (ksup >= 1) then + if (TFocean(ksup, iCell) == invalid_value_TF) then + call mpas_log_write("iceshelf_melt_ismip6: Invalid value for TFocean. " // & + "ksup=$i TFocean(ksup, iCell)=$r indexToCellID=$i", MPAS_LOG_ERR, & + intArgs=(/ksup, indexToCellID(iCell)/), & + realArgs=(/TFocean(ksup, iCell) /) ) + err = ior(err, 1) + endif + endif + if (kinf <= nOceanLayers) then + if (TFocean(kinf, iCell) == invalid_value_TF) then + call mpas_log_write("iceshelf_melt_ismip6: Invalid value for TFocean. " // & + "kinf=$i TFocean(kinf,iCell)=$r indexToCellID=$i", MPAS_LOG_ERR, & + intArgs=(/kinf, indexToCellID(iCell)/), & + realArgs=(/TFocean(kinf, iCell) /) ) + err = ior(err, 1) + endif + endif + + if (ksup == 0) then + ! For depths shallower than shallowest layer center, use shallowest layer + TFdraft(iCell) = TFocean(1, iCell) + elseif (ksup == nOceanLayers) then + ! for depths below the deepest layer center, use deepest layer corrected for Tfreeze + TFdraft(iCell) = TFocean(nOceanLayers, iCell) - & + (zOcean(nOceanLayers) - lowerSurface(iCell)) * oceanFreezingTempDepthDependence + else + ! for depths between the first and last layer centers, linearly interpolate + TFdraft(iCell) = ( (zOcean(ksup)-lowerSurface(iCell)) * TFocean(kinf, iCell) & + + (lowerSurface(iCell)-zOcean(kinf)) * TFocean(ksup, iCell) ) / (zOcean(ksup)-zOcean(kinf)) + endif ! 2 - Mean Thermal forcing in individual basins (NB: fortran norm while basins start at zero): mean_TF(basinNumber(iCell)+1) = mean_TF(basinNumber(iCell)+1) + areaCell(iCell) * TFdraft(iCell) diff --git a/components/mpas-albany-landice/src/mode_forward/mpas_li_ocean_extrap.F b/components/mpas-albany-landice/src/mode_forward/mpas_li_ocean_extrap.F new file mode 100644 index 000000000000..de501d629605 --- /dev/null +++ b/components/mpas-albany-landice/src/mode_forward/mpas_li_ocean_extrap.F @@ -0,0 +1,601 @@ +! Copyright (c) 2013-2018, Los Alamos National Security, LLC (LANS) +! and the University Corporation for Atmospheric Research (UCAR). +! +! Unless noted otherwise source code is licensed under the BSD license. +! Additional copyright and license information can be found in the LICENSE file +! distributed with this code, or at http://mpas-dev.github.com/license.html +! + +!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +! +! li_ocean_extrap +! +!> \MPAS land-ice ocean-data extrapolation driver +!> \author Holly Han +!> \date January 2022 +!> \details +!> This module contains the routines for extrapolating +!> ocean data (e.g., temperature, salinity, thermal forcing) +!> into ice draft +! +!----------------------------------------------------------------------- + +module li_ocean_extrap + + use mpas_derived_types + use mpas_pool_routines + use mpas_dmpar + use mpas_log + use li_mask + use li_setup + + implicit none + private + + !-------------------------------------------------------------------- + ! Public parameters + !-------------------------------------------------------------------- + + !-------------------------------------------------------------------- + ! Public member functions + !-------------------------------------------------------------------- + + public :: li_ocean_extrap_solve + + !-------------------------------------------------------------------- + ! Private module variables + !-------------------------------------------------------------------- + +!*********************************************************************** + +contains + +!*********************************************************************** +! +! routine li_ocean_extrap_solve +! +!> \brief Initializes ocean extrapolation scheme +!> \author Holly Han +!> \date 12 Jan 2023 +!> \details +!> This routine performs horizontal and vertical extrapolation +!> of ocean data (e.g., temperature, salinity, thermal forcing) +! +!----------------------------------------------------------------------- + + subroutine li_ocean_extrap_solve(domain, err) + use li_calving, only: li_flood_fill + + !----------------------------------------------------------------- + ! input variables + !----------------------------------------------------------------- + + !----------------------------------------------------------------- + ! input/output variables + !----------------------------------------------------------------- + type (domain_type), intent(inout) :: domain !< Input/Output: domain object + + !----------------------------------------------------------------- + ! output variables + !----------------------------------------------------------------- + integer, intent(out) :: err !< Output: error flag + + !----------------------------------------------------------------- + ! local variables + !----------------------------------------------------------------- + logical, pointer :: config_ocean_data_extrapolation + real(kind=RKIND), pointer :: config_sea_level, invalid_value_TF + type (block_type), pointer :: block + type (mpas_pool_type), pointer :: scratchPool, geometryPool, meshPool, extrapOceanDataPool + real (kind=RKIND) :: layerTop + real (kind=RKIND), dimension(:,:), pointer :: TFocean, TFoceanOld + real (kind=RKIND), dimension(:,:), pointer :: ismip6shelfMelt_3dThermalForcing, ismip6shelfMelt_zBndsOcean + real (kind=RKIND), dimension(:), pointer :: ismip6shelfMelt_zOcean + real (kind=RKIND), dimension(:), pointer :: thickness, bedTopography + integer, dimension(:,:), pointer :: orig3dOceanMask + integer, dimension(:,:), pointer :: validOceanMask, availOceanMask !masks to pass to flood-fill routine + integer, dimension(:), pointer :: seedOceanMaskHoriz, growOceanMaskHoriz + integer, dimension(:), allocatable :: seedOceanMaskHorizOld + integer, pointer :: nCells, nCellsSolve, nISMIP6OceanLayers, nCellsExtra + integer, dimension(:), pointer :: cellMask, nEdgesOnCell + integer, dimension(:,:), pointer :: cellsOnCell + integer :: iCell, jCell, iLayer, iNeighbor, iter, err_tmp + integer :: GlobalLoopCount, newMaskCountGlobal + type (field1dInteger), pointer :: seedMaskField + type (field1dInteger), pointer :: growMaskField + integer, dimension(:), pointer :: connectedMarineMask, growMask !masks to pass to flood-fill routine + + ! No init is needed. + err = 0 + err_tmp = 0 + + call mpas_pool_get_config(liConfigs, 'config_ocean_data_extrapolation', config_ocean_data_extrapolation) + + if ( config_ocean_data_extrapolation ) then + ! call the extrapolation scheme + call mpas_log_write('ocean data will be extrapolated into the MALI ice draft') + block => domain % blocklist + + ! initialize the ocean data and mask fields + call mpas_pool_get_config(liConfigs, 'config_sea_level', config_sea_level) + call mpas_pool_get_config(liConfigs, 'config_ocean_data_extrap_ncells_extra', nCellsExtra) + call mpas_pool_get_subpool(block % structs, 'mesh', meshPool) + call mpas_pool_get_subpool(block % structs, 'geometry', geometryPool) + call mpas_pool_get_subpool(block % structs, 'extrapOceanData', extrapOceanDataPool) + call mpas_pool_get_dimension(meshPool, 'nISMIP6OceanLayers', nISMIP6OceanLayers) + call mpas_pool_get_dimension(meshPool, 'nCells', nCells) + call mpas_pool_get_dimension(meshPool, 'nCellsSolve', nCellsSolve) + call mpas_pool_get_array(meshPool, 'cellsOnCell', cellsOnCell) + call mpas_pool_get_array(meshPool, 'nEdgesOnCell', nEdgesOnCell) + call mpas_pool_get_array(geometryPool, 'cellMask', cellMask) + call mpas_pool_get_array(geometryPool, 'thickness', thickness) + call mpas_pool_get_array(geometryPool, 'bedTopography', bedTopography) + call mpas_pool_get_array(geometryPool, 'ismip6shelfMelt_3dThermalForcing', ismip6shelfMelt_3dThermalForcing) + call mpas_pool_get_array(extrapOceanDataPool, 'orig3dOceanMask', orig3dOceanMask) + call mpas_pool_get_array(extrapOceanDataPool, 'validOceanMask', validOceanMask) + call mpas_pool_get_array(extrapOceanDataPool, 'availOceanMask', availOceanMask) + call mpas_pool_get_array(extrapOceanDataPool, 'seedOceanMaskHoriz', seedOceanMaskHoriz) + call mpas_pool_get_array(extrapOceanDataPool, 'growOceanMaskHoriz', growOceanMaskHoriz) + call mpas_pool_get_array(extrapOceanDataPool, 'TFoceanOld', TFoceanOld) + call mpas_pool_get_array(extrapOceanDataPool, 'TFocean', TFocean) + call mpas_pool_get_config(liConfigs, 'config_invalid_value_TF', invalid_value_TF) + + ! create a 2D mask based on the open ocean + floating ice + grounded ice mask to be used for defining the location of the n-extra cells into the grounded ice + allocate(seedOceanMaskHorizOld(nCells+1)) + seedOceanMaskHorizOld(:) = 0 + seedOceanMaskHoriz(:) = 0 + growOceanMaskHoriz(:) = 0 + ! define seedOceanMaskHoriz and growOceanMaskHoriz for horizontal floodfill + do iCell = 1, nCellsSolve + if ( .not. li_mask_is_grounded_ice(cellMask(iCell)) .and. bedTopography(iCell) < config_sea_level ) then + seedOceanMaskHoriz(iCell) = 1 + endif + if ( bedTopography(iCell) < config_sea_level ) then + growOceanMaskHoriz(iCell) = 1 + endif + enddo + + ! Start horizontal floodfill to define locations of n-extra cells + seedOceanMaskHorizOld(:) = seedOceanMaskHoriz(:) + ! go through the loop to get nCells extra into grounded ice + do iter = 1, nCellsExtra + do iCell = 1, nCellsSolve + if ( growOceanMaskHoriz(iCell) == 1 .and. seedOceanMaskHorizOld(iCell) == 0 ) then + do iNeighbor = 1, nEdgesOnCell(iCell) + jCell = cellsOnCell(iNeighbor, iCell) + if ( seedOceanMaskHorizOld(jCell) == 1 ) then + seedOceanMaskHoriz(iCell) = 1 + endif + enddo + endif + enddo + seedOceanMaskHorizOld(:) = seedOceanMaskHoriz(:) + ! Update halos + ! Note: halo updates could be skipped for a number of iterations equal to the + ! number of halo rows as a possible performance improvement in the future. + ! (And the loop above would need to be changed to from nCellsSolve to nCells) + call mpas_timer_start("halo updates") + call mpas_dmpar_field_halo_exch(domain, 'growOceanMaskHoriz') + call mpas_dmpar_field_halo_exch(domain, 'seedOceanMaskHoriz') + call mpas_timer_stop("halo updates") + enddo + deallocate(seedOceanMaskHorizOld) + + ! Calculate mask of connected ocean + call mpas_pool_get_subpool(domain % blocklist % structs, 'scratch', scratchPool) + call mpas_pool_get_field(scratchPool, 'seedMask', seedMaskField) + call mpas_allocate_scratch_field(seedMaskField, single_block_in = .true.) + connectedMarineMask => seedMaskField % array + connectedMarineMask(:) = 0 + call mpas_pool_get_field(scratchPool, 'growMask', growMaskField) + call mpas_allocate_scratch_field(growMaskField, single_block_in = .true.) + growMask => growMaskField % array + growMask(:) = 0 + + do iCell = 1, nCells + ! seedMask = open ocean cells in contact with the domain boundary + if ((bedTopography(iCell) < config_sea_level) .and. (thickness(iCell) == 0.0_RKIND)) then + do iNeighbor = 1, nEdgesOnCell(iCell) + if (cellsOnCell(iNeighbor, iCell) == nCells + 1) then + connectedMarineMask(iCell) = 1 + exit ! no need to keep checking neighbors + endif + enddo + endif + ! growMask - all marine cells + if (bedTopography(iCell) < config_sea_level) then + growMask(iCell) = 1 + endif + enddo + ! now create mask of all marine locations connected to open ocean - to be used below to screen out lakes + call li_flood_fill(connectedMarineMask, growMask, domain) + + ! make it a 3D mask based on the topography (loop through nISMIP6OceanLayers) + call mpas_pool_get_dimension(meshPool, 'nISMIP6OceanLayers', nISMIP6OceanLayers) + call mpas_pool_get_array(geometryPool, 'ismip6shelfMelt_zOcean', ismip6shelfMelt_zOcean) + call mpas_pool_get_array(geometryPool, 'ismip6shelfMelt_zBndsOcean', ismip6shelfMelt_zBndsOcean) + ! check for valid data + do iLayer = 1, nISMIP6OceanLayers + if (ismip6shelfMelt_zOcean(iLayer) >= 0.0_RKIND) then + call mpas_log_write("ismip6shelfMelt_zOcean has invalid value of $r in layer $i", MPAS_LOG_ERR, & + realArgs=(/ismip6shelfMelt_zOcean(iLayer)/), intArgs=(/iLayer/)) + call mpas_log_write("ismip6shelfMelt_zOcean must have negative values because they represent " // & + "depths below sea level.", MPAS_LOG_ERR) + err = ior(err, 1) + endif + if ((ismip6shelfMelt_zBndsOcean(1,iLayer) > 0.0_RKIND) .or. & + (ismip6shelfMelt_zBndsOcean(1,iLayer) < ismip6shelfMelt_zOcean(iLayer))) then + call mpas_log_write("ismip6shelfMelt_zBndsOcean(1,:) has invalid value of $r in layer $i", MPAS_LOG_ERR, & + realArgs=(/ismip6shelfMelt_zBndsOcean(1,iLayer)/), intArgs=(/iLayer/)) + call mpas_log_write("ismip6shelfMelt_zBndsOcean(1,:) must be less than or equal to zero " // & + "because it represents the upper bound of an ocean layer", MPAS_LOG_ERR) + err = ior(err, 1) + endif + if ((ismip6shelfMelt_zBndsOcean(2,iLayer) >= 0.0_RKIND) .or. & + (ismip6shelfMelt_zBndsOcean(2,iLayer) > ismip6shelfMelt_zOcean(iLayer))) then + call mpas_log_write("ismip6shelfMelt_zBndsOcean(2,:) has invalid value of $r in layer $i", MPAS_LOG_ERR, & + realArgs=(/ismip6shelfMelt_zBndsOcean(2,iLayer)/), intArgs=(/iLayer/)) + call mpas_log_write("ismip6shelfMelt_zBndsOcean(2,:) must be less than zero " // & + "because it represents the lower bound of an ocean layer", MPAS_LOG_ERR) + err = ior(err, 1) + endif + enddo + availOceanMask(:,:) = 0 + do iCell = 1, nCells + do iLayer = 1, nISMIP6OceanLayers + layerTop = ismip6shelfMelt_zBndsOcean(1, iLayer) + if ( (seedOceanMaskHoriz(iCell) == 1) .and. (connectedMarineMask(iCell) == 1)) then + if (bedTopography(iCell) < layerTop) then + availOceanMask(iLayer,iCell) = 1 + else + ! keep the first layer below the seafloor in the region to be filled + ! this ensures linear interpolation from above and below the seafloor is possible + availOceanMask(iLayer,iCell) = 1 + exit ! stop looping over levels after we've included the first level below the seafloor + endif + endif + enddo + enddo + + ! Make a copy of original mask to use for extending the mask during extrapolation + validOceanMask(:,:) = orig3dOceanMask(:,:) + + ! initialize the TF field + TFocean(:,:) = ismip6shelfMelt_3dThermalForcing(:,:) * validOceanMask(:,:) + ! initialize the invalid data locations with fill value + do iCell = 1, nCellsSolve + do iLayer = 1, nISMIP6OceanLayers + if ((connectedMarineMask(iCell) == 0) .and. (bedTopography(iCell) < config_sea_level)) then + ! Don't assign invalid value to lakes/inland seas disconnected from global ocean + ! Let them retain the existing value: + ! This will take on the valid ocean data value where it exists or + ! zero where valid ocean data does not exist + elseif (validOceanMask(iLayer,iCell) == 0) then + ! everywhere else where valid ocean data does not exist, insert invalid value outside of validOceanMask + TFocean(iLayer,iCell) = invalid_value_TF + endif + enddo + enddo + + ! deallocate scratch fields used for flood fill + call mpas_deallocate_scratch_field(seedMaskField, single_block_in=.true.) + call mpas_deallocate_scratch_field(growMaskField, single_block_in=.true.) + + ! flood-fill the valid ocean mask and TF field through + ! horizontal and vertial extrapolation + ! get initial 3D valid data based on the original ISMIP6 field + newMaskCountGlobal = 1 + GlobalLoopCount = 0 + do while (newMaskCountGlobal > 0) + newMaskCountGlobal = 0 + GlobalLoopCount = GlobalLoopCount + 1 + if (GlobalLoopCount > 20) then + ! There will only be an additional time through this loop for each sill behind a previous sill + ! so it should not need to alternate very many times + call mpas_log_write("Ocean extrapolation has alternated between horizontal and vertical " // & + "extrapolation more than $i times. Aborting", MPAS_LOG_ERR, intArgs=(/GlobalLoopCount/)) + err = ior(err, 1) + endif + ! call the horizontal extrapolation routine + call mpas_timer_start("horizontal scheme") + call horizontal_extrapolation(domain, availOceanMask, validOceanMask, orig3dOceanMask, TFocean, err_tmp) + err = ior(err, err_tmp) + call mpas_timer_stop("horizontal scheme") + ! call the vertical extrapolation routine + call mpas_timer_start("vertical scheme") + call vertical_extrapolation(domain, availOceanMask, validOceanMask, newMaskCountGlobal, TFocean, err_tmp) + err = ior(err, err_tmp) + call mpas_timer_stop("vertical scheme") + + if (err > 0) then + call mpas_log_write("Ocean extrapolation main iteration loop has encountered an error", MPAS_LOG_ERR) + return + endif + enddo + + ! Reassign extrapolated TF back to primary TF field + ismip6shelfMelt_3dThermalForcing(:,:) = TFocean(:,:) + endif + !-------------------------------------------------------------------- + + end subroutine li_ocean_extrap_solve +!----------------------------------------------------------------------- + + +!*********************************************************************** +!*********************************************************************** +! Private subroutines: +!*********************************************************************** +!*********************************************************************** + + + +!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +! +! ! routine horizontal_extrapolation +! +!> \brief Extrapolate validOceanMask horizontally +!> \author Holly Kyeore Han +!> \date November 2023 +!> \details +!> This routine extrapolates takes the initialized availOceanMask +!> and validOceanMask and extrapolates validOceanMask in horizontal +!> direction until the local new mask count stops updating. +!> The output of the routine is an updated validOceanMask field and +!> newMaskCountLocal. + +!----------------------------------------------------------------------- + + subroutine horizontal_extrapolation(domain, availOceanMask, validOceanMask, validOceanMaskOrig, TFocean, err) + + !----------------------------------------------------------------- + ! input variables + !----------------------------------------------------------------- + integer, dimension(:,:), pointer, intent(in) :: availOceanMask, validOceanMaskOrig + + !----------------------------------------------------------------- + ! input/output variables + !----------------------------------------------------------------- + type (domain_type), intent(inout) :: domain !< Input/Output: domain object + integer, dimension(:,:), pointer, intent(inout) :: validOceanMask + integer, intent(inout) :: err !< Output: error flag + real (kind=RKIND), dimension(:,:), pointer, intent(inout) :: TFocean + + !----------------------------------------------------------------- + ! output variables + !----------------------------------------------------------------- + + !----------------------------------------------------------------- + ! local variables + !----------------------------------------------------------------- + type (block_type), pointer :: block + type (mpas_pool_type), pointer :: scratchPool, geometryPool, meshPool, extrapOceanDataPool + real (kind=RKIND) :: layerTop, TFsum, areaSum, weightCellLocal + real (kind=RKIND), pointer :: weightCell + integer, dimension(:,:), allocatable :: validOceanMaskOld + real (kind=RKIND), dimension(:,:), pointer :: ismip6shelfMelt_3dThermalForcing + real (kind=RKIND), dimension(:,:), pointer :: TFoceanOld + real (kind=RKIND), dimension(:), pointer :: thickness, bedTopography, areaCell + integer, pointer :: nCells, nCellsSolve, nISMIP6OceanLayers, nCellsExtra + integer, dimension(:), pointer :: cellMask, nEdgesOnCell + integer, dimension(:), pointer :: indexToCellID + integer, dimension(:,:), pointer :: cellsOnCell + integer :: iCell, jCell, iLayer, iNeighbor, iter + integer :: localLoopCount + integer :: nValidNeighb, newValidCount, newMaskCountLocalAccum, newMaskCountGlobal + integer :: newMaskCountTotal + + err = 0 + + ! initialize the ocean data and mask fields + block => domain % blocklist + call mpas_pool_get_config(liConfigs, 'config_ocean_data_extrap_ncells_extra', nCellsExtra) + call mpas_pool_get_config(liConfigs,'config_weight_value_cell', weightCell) + call mpas_pool_get_subpool(block % structs, 'mesh', meshPool) + call mpas_pool_get_subpool(block % structs, 'geometry', geometryPool) + call mpas_pool_get_subpool(block % structs, 'extrapOceanData', extrapOceanDataPool) + call mpas_pool_get_dimension(meshPool, 'nISMIP6OceanLayers', nISMIP6OceanLayers) + call mpas_pool_get_dimension(meshPool, 'nCells', nCells) + call mpas_pool_get_dimension(meshPool, 'nCellsSolve', nCellsSolve) + call mpas_pool_get_array(meshPool, 'cellsOnCell', cellsOnCell) + call mpas_pool_get_array(meshPool, 'areaCell', areaCell) + call mpas_pool_get_array(meshPool, 'nEdgesOnCell', nEdgesOnCell) + call mpas_pool_get_array(meshPool, 'indexToCellID', indexToCellID) + call mpas_pool_get_array(geometryPool, 'cellMask', cellMask) + call mpas_pool_get_array(geometryPool, 'thickness', thickness) + call mpas_pool_get_array(geometryPool, 'bedTopography', bedTopography) + call mpas_pool_get_array(extrapOceanDataPool, 'TFoceanOld', TFoceanOld) + + !TFoceanOld(:,:) = 1.0 ! for debugging, set TF to ones to make it easy to verify the horizonal/vertical averaging + ! perform horizontal extrapolation until the validOceanMask is unchanged + allocate(validOceanMaskOld(nISMIP6OceanLayers,nCells+1)) + validOceanMaskOld(:,:) = validOceanMask(:,:) + TFoceanOld(:,:) = TFocean(:,:) + + ! initialize the local loop and count for validOceanMask + localLoopCount = 0 + newMaskCountTotal = 0 + newMaskCountGlobal = 1 + call mpas_log_write('Weight given to the cell with valid data from extrapolation: $r', realArgs=(/weightCell/)) + do while ( newMaskCountGlobal > 0 ) + localLoopCount = localLoopCount + 1 + newMaskCountLocalAccum = 0 + do iCell = 1, nCellsSolve + do iLayer = 1, nISMIP6OceanLayers + if ( (availOceanMask(iLayer,iCell) == 1) .and. (validOceanMaskOrig(iLayer,iCell) == 0) ) then + TFsum = 0.0 + areaSum = 0.0 + newValidCount = 0 + nValidNeighb = 0 + do iNeighbor = 1, nEdgesOnCell(iCell) + jCell = cellsOnCell(iNeighbor, iCell) + if ( validOceanMaskOld(iLayer,jCell) == 1 ) then + if ( TFoceanOld(iLayer,jCell) > 1.0e6_RKIND) then + ! raise error if an invalid ocean data value is used + call mpas_log_write("ocean data value used for extrapolation is invalid " // & + "in horizontal_extrapolation: cell id=$i, iLayer=$i, TF=$r", & + MPAS_LOG_ERR, intArgs=(/indexToCellID(jCell), iLayer/), realArgs=(/TFoceanOld(iLayer,jCell)/)) + err = ior(err,1) + else + TFsum = TFsum + (TFoceanOld(iLayer,jCell) * areaCell(jCell)) + endif + areaSum = areaSum + areaCell(jCell) + nValidNeighb = nValidNeighb + 1 + endif + enddo + if ( validOceanMaskOld(iLayer,iCell) == 0 .and. nValidNeighb > 0 ) then + ! if current cell is not valid, set its weight to zero + weightCellLocal = 0.0_RKIND + validOceanMask(iLayer,iCell) = 1 + newValidCount = 1 + else + weightCellLocal = weightCell + endif + ! perform area-weighted averaging of the thermal forcing field + if ( nValidNeighb == 0 ) then + TFocean(iLayer,iCell) = TFoceanOld(iLayer,iCell) + else + TFocean(iLayer,iCell) = ( weightCellLocal * TFoceanOld(iLayer,iCell) * areaCell(iCell) + & + & ((1.0_RKIND - weightCellLocal) * (TFsum / nValidNeighb)) ) / & + & ( weightCellLocal * areaCell(iCell) + & + & (1.0_RKIND - weightCellLocal) * (areaSum / nValidNeighb) ) + endif + ! Accumulate cells added locally until we do the next global reduce + newMaskCountLocalAccum = newMaskCountLocalAccum + newValidCount + endif + enddo + enddo + + ! update halo for validOceanMask and ocean data + call mpas_timer_start("halo updates") + call mpas_dmpar_field_halo_exch(domain, 'validOceanMask') + call mpas_dmpar_field_halo_exch(domain, 'TFocean') + call mpas_timer_stop("halo updates") + + validOceanMaskOld(:,:) = validOceanMask(:,:) + TFoceanOld(:,:) = TFocean(:,:) + + ! update count of cells added to mask globally + call mpas_dmpar_sum_int(domain % dminfo, newMaskCountLocalAccum, newMaskCountGlobal) + newMaskCountTotal = newMaskCountTotal + newMaskCountGlobal + !call mpas_log_write('Horizontal extrap: Added total $i new cells to validOceanMask', intArgs=(/newMaskCountGlobal/)) + enddo + call mpas_log_write('Horizontal extrapolation done after $i iterations. Added total of $i cells across all processors', & + intArgs=(/localLoopCount, newMaskCountTotal/)) + deallocate(validOceanMaskOld) + + end subroutine horizontal_extrapolation +!----------------------------------------------------------------------- + + +!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +! +! ! routine vertical_extrapolation +! +!> \brief Extrapolate validOceanMask vertically +!> \author Holly Kyeore Han +!> \date November 2023 +!> \details +!> This routine extrapolates the horizontally extrapolated +!> validOceanMask through the vertical layers of the ocean. +!> The vertical extrapolation is completed once local new mask count +!> stops updating. The output of the routine is an updated +!> validOceanMask, thermal forcing fields and newMaskCountLocal. + +!----------------------------------------------------------------------- + + subroutine vertical_extrapolation(domain, availOceanMask, validOceanMask, newMaskCountGlobal, TFocean, err) + + use li_constants, only: oceanFreezingTempDepthDependence + + !----------------------------------------------------------------- + ! input variables + !----------------------------------------------------------------- + integer, dimension(:,:), pointer, intent(in) :: availOceanMask + + !----------------------------------------------------------------- + ! input/output variables + !----------------------------------------------------------------- + type (domain_type), intent(inout) :: domain !< Input/Output: domain object + integer, dimension(:,:), pointer, intent(inout) :: validOceanMask + integer, intent(inout) :: err !< Output: error flag + real (kind=RKIND), dimension(:,:), pointer, intent(inout) :: TFocean + + !----------------------------------------------------------------- + ! output variables + !----------------------------------------------------------------- + integer, intent(out) :: newMaskCountGlobal + + !----------------------------------------------------------------- + ! local variables + !----------------------------------------------------------------- + type (block_type), pointer :: block + type (mpas_pool_type), pointer :: scratchPool, geometryPool, meshPool, extrapOceanDataPool + real (kind=RKIND) :: layerTop, TFsum, areaSum + real (kind=RKIND), dimension(:), pointer :: ismip6shelfMelt_zOcean + real (kind=RKIND), dimension(:), pointer :: thickness, bedTopography, areaCell + integer, pointer :: nCells, nCellsSolve, nISMIP6OceanLayers + integer, dimension(:), pointer :: cellMask, nEdgesOnCell + integer, dimension(:), pointer :: indexToCellID + integer, dimension(:,:), pointer :: cellsOnCell + integer :: iCell, jCell, iLayer, iNeighbor, iter + integer :: localLoopCount, newMaskCountLocalAccum + + err = 0 + + ! initialize the ocean data and mask fields + block => domain % blocklist + call mpas_pool_get_subpool(block % structs, 'mesh', meshPool) + call mpas_pool_get_subpool(block % structs, 'geometry', geometryPool) + call mpas_pool_get_subpool(block % structs, 'extrapOceanData', extrapOceanDataPool) + call mpas_pool_get_dimension(meshPool, 'nISMIP6OceanLayers', nISMIP6OceanLayers) + call mpas_pool_get_dimension(meshPool, 'nCells', nCells) + call mpas_pool_get_dimension(meshPool, 'nCellsSolve', nCellsSolve) + call mpas_pool_get_array(meshPool, 'cellsOnCell', cellsOnCell) + call mpas_pool_get_array(meshPool, 'areaCell', areaCell) + call mpas_pool_get_array(meshPool, 'nEdgesOnCell', nEdgesOnCell) + call mpas_pool_get_array(meshPool, 'indexToCellID', indexToCellID) + call mpas_pool_get_array(geometryPool, 'cellMask', cellMask) + call mpas_pool_get_array(geometryPool, 'thickness', thickness) + call mpas_pool_get_array(geometryPool, 'bedTopography', bedTopography) + call mpas_pool_get_array(geometryPool, 'ismip6shelfMelt_zOcean', ismip6shelfMelt_zOcean) + + ! initialize the local loop and count for validOceanMask + newMaskCountGlobal = 0 + newMaskCountLocalAccum = 0 + do iCell = 1, nCellsSolve + do iLayer = 2, nISMIP6OceanLayers + if ( (availOceanMask(iLayer,iCell) == 1) .and. (validOceanMask(iLayer,iCell) == 0) ) then + if ( validOceanMask(iLayer-1,iCell) == 1 ) then + if (TFocean(iLayer-1,iCell) > 1.0e6_RKIND) then + ! raise error if an invalid ocean data value is used + call mpas_log_write("ocean data value used for extrapolation is invalid " // & + "in vertical_extrapolation: cell id=$i, iLayer=$i, TF=$r", & + MPAS_LOG_ERR, intArgs=(/indexToCellID(iCell), iLayer-1/), realArgs=(/TFocean(iLayer-1,iCell)/)) + err = ior(err,1) + else + TFocean(iLayer,iCell) = TFocean(iLayer-1,iCell) - & + (ismip6shelfMelt_zOcean(iLayer-1) - ismip6shelfMelt_zOcean(iLayer)) * & + oceanFreezingTempDepthDependence + endif + validOceanMask(iLayer,iCell) = 1 + newMaskCountLocalAccum = newMaskCountLocalAccum + 1 + endif + endif + enddo + enddo + + ! update halo for validOceanMask and ocean data + call mpas_timer_start("halo updates") + call mpas_dmpar_field_halo_exch(domain, 'validOceanMask') + call mpas_dmpar_field_halo_exch(domain, 'TFocean') + call mpas_timer_stop("halo updates") + ! update count of cells added to mask globally + call mpas_dmpar_sum_int(domain % dminfo, newMaskCountLocalAccum, newMaskCountGlobal) + call mpas_log_write('Vertical extrap: Added total $i new cells to the validOceanMask', intArgs=(/newMaskCountGlobal/)) + + end subroutine vertical_extrapolation +!----------------------------------------------------------------------- +end module li_ocean_extrap + +!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| diff --git a/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F b/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F index b722459761eb..bf9616b4339b 100644 --- a/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F +++ b/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F @@ -59,6 +59,9 @@ module li_subglacial_hydro ! Minimum gradMagPhiBaseEdge and gradMagPhiEdge allowed before all dependent variables are zeroed out real(kind=RKIND), parameter :: SMALL_GRADPHI = 1.0e-6_RKIND + !Minimum outflowing hydropotential slope applied at grounding line + real(kind=RKIND), parameter :: MIN_PHISLOPE_GL = 1e-10_RKIND + !*********************************************************************** contains @@ -107,6 +110,7 @@ subroutine li_SGH_init(domain, err) real (kind=RKIND), dimension(:), pointer :: thickness real (kind=RKIND), dimension(:), pointer :: bedTopography real (kind=RKIND), dimension(:), pointer :: iceThicknessHydro + real (kind=RKIND), dimension(:), pointer :: hydropotential integer, dimension(:), pointer :: cellMask real (kind=RKIND), pointer :: tillMax real (kind=RKIND), pointer :: rhoi, rhoo @@ -199,18 +203,25 @@ subroutine li_SGH_init(domain, err) call mpas_pool_get_subpool(block % structs, 'geometry', geometryPool) call mpas_pool_get_array(hydroPool, 'waterPressure', waterPressure) + call mpas_pool_get_array(hydroPool, 'hydropotential', hydropotential) call mpas_pool_get_array(hydroPool, 'iceThicknessHydro', iceThicknessHydro) - - waterPressure = max(0.0_RKIND, waterPressure) - waterPressure = min(waterPressure, rhoi * gravity * iceThicknessHydro) - ! set pressure correctly under floating ice and open ocean call mpas_pool_get_array(geometryPool, 'cellMask', cellMask) call mpas_pool_get_array(geometryPool, 'bedTopography', bedTopography) - where ( (li_mask_is_floating_ice(cellMask)) .or. & - ( (.not. li_mask_is_ice(cellMask)) .and. (bedTopography < config_sea_level) ) ) - waterPressure = rhoo * gravity * (config_sea_level - bedTopography) + + waterPressure = max(0.0_RKIND, waterPressure) + where (li_mask_is_grounded_ice(cellMask)) + waterPressure = min(waterPressure, rhoi * gravity * iceThicknessHydro) end where - + + ! set pressure and hydropotential correctly on ice-free land and in ocean + where ((.not. (li_mask_is_grounded_ice(cellMask))) .and. (bedTopography > config_sea_level)) + waterPressure = 0.0_RKIND + hydropotential = rho_water * gravity * bedTopography + elsewhere ((.not. (li_mask_is_grounded_ice(cellMask))) .and. (bedTopography <= config_sea_level)) + waterPressure = gravity * rhoo * (config_sea_level - bedTopography) + hydropotential = 0.0_RKIND + end where + ! Initialize diagnostic pressure variables call calc_pressure_diag_vars(block, err_tmp) err = ior(err, err_tmp) @@ -303,6 +314,8 @@ subroutine li_SGH_solve(domain, err) real (kind=RKIND), dimension(:), pointer :: waterVelocity real (kind=RKIND), dimension(:), pointer :: waterVelocityCellX real (kind=RKIND), dimension(:), pointer :: waterVelocityCellY + real (kind=RKIND), dimension(:), pointer :: effectivePressureSGH + real (kind=RKIND), dimension(:), pointer :: effectivePressure real (kind=RKIND), dimension(:), allocatable :: cellJunk integer, dimension(:), pointer :: nEdgesOnCell integer, dimension(:,:), pointer :: edgesOnCell @@ -317,6 +330,7 @@ subroutine li_SGH_solve(domain, err) integer, pointer :: nCells integer :: iCell, iEdge, iEdgeOnCell real (kind=RKIND) :: timeLeft ! subcycling time remaining until master dt is reached + real (kind=RKIND) :: deltatSGHaccumulated integer :: numSubCycles ! number of subcycles integer :: err_tmp @@ -370,6 +384,19 @@ subroutine li_SGH_solve(domain, err) call mpas_pool_get_array(meshPool, 'deltat', masterDeltat) timeLeft = masterDeltat ! in seconds numSubCycles = 0 + + block => domain % blocklist + do while (associated(block)) + + call mpas_pool_get_subpool(block % structs, 'hydro', hydroPool) + call mpas_pool_get_array(hydroPool, 'effectivePressure', effectivePressure) + effectivePressure = 0.0_RKIND + deltatSGHaccumulated = 0.0_RKIND + + block => block % next + end do + + ! ============= ! ============= ! ============= @@ -669,7 +696,22 @@ subroutine li_SGH_solve(domain, err) call mpas_dmpar_field_halo_exch(domain, 'waterPressureSmooth') call mpas_timer_stop("halo updates") + !Average effectivePressureSGH over coupling interval for use in dynamics model + block => domain % blocklist + do while (associated(block)) + + call mpas_pool_get_subpool(block % structs, 'hydro', hydroPool) + call mpas_pool_get_array(hydroPool, 'effectivePressureSGH', effectivePressureSGH) + call mpas_pool_get_array(hydroPool, 'effectivePressure', effectivePressure) + call mpas_pool_get_array(hydroPool, 'deltatSGH', deltatSGH) + + effectivePressure = (effectivePressure * deltatSGHaccumulated + effectivePressureSGH * deltatSGH) / (deltatSGHaccumulated + deltatSGH) + + block => block % next + end do + deltatSGHaccumulated = deltatSGHaccumulated + deltatSGH + ! ============= ! ============= ! ============= @@ -821,6 +863,7 @@ subroutine calc_edge_quantities(block, err) real (kind=RKIND), dimension(:), pointer :: waterFluxDiffu real (kind=RKIND), dimension(:), pointer :: waterPressureSmooth integer, dimension(:), pointer :: hydroMarineMarginMask + integer, dimension(:), pointer :: hydroTerrestrialMarginMask integer, dimension(:), pointer :: waterFluxMask integer, dimension(:,:), pointer :: edgeSignOnCell integer, dimension(:), pointer :: cellMask @@ -897,6 +940,7 @@ subroutine calc_edge_quantities(block, err) call mpas_pool_get_array(hydroPool, 'waterFluxDiffu', waterFluxDiffu) call mpas_pool_get_array(hydroPool, 'waterFluxMask', waterFluxMask) call mpas_pool_get_array(hydroPool, 'hydroMarineMarginMask', hydroMarineMarginMask) + call mpas_pool_get_array(hydroPool, 'hydroTerrestrialMarginMask', hydroTerrestrialMarginMask) call mpas_pool_get_array(geometryPool, 'edgeMask', edgeMask) call mpas_pool_get_array(hydroPool, 'waterPressureSmooth', waterPressureSmooth) @@ -920,53 +964,58 @@ subroutine calc_edge_quantities(block, err) waterPressureSlopeNormal(iEdge) = (waterPressureSmooth(cell2) - waterPressureSmooth(cell1)) / dcEdge(iEdge) end do - ! At boundaries of hydro domain, disallow inflow. Allow outflow if hydropotential gradient requires it. + ! At terrestrial margin, ignore the downslope bed topography gradient. Including it can lead to unrealistically large + ! hydropotential gradients and unstable channel growth. + ! The hydropotential at the terrestrial margin should be determined by the geometry + ! at the edge of the cell in a 1-sided sense. For hydropotentialBase = rho*g*Zb + Pw, this means hydropotentialBaseSlopeNormal + ! at the terrestrial margin is equal to Pw/dcEdge. + ! This one-sided implementation also creates outflowing conditions at terrestrial boundary do iEdge = 1, nEdges - if ( (li_mask_is_margin(edgeMask(iEdge)) .and. li_mask_is_grounded_ice(edgeMask(iEdge))) .or. & - (hydroMarineMarginMask(iEdge)==1)) then + if (hydroTerrestrialMarginMask(iEdge) == 1) then cell1 = cellsOnEdge(1, iEdge) cell2 = cellsOnEdge(2, iEdge) - if (li_mask_is_grounded_ice(cellMask(cell1))) then ! cell2 is the cell outside the hydro domain - hydropotentialBaseSlopeNormal(iEdge) = min(0.0_RKIND, hydropotentialBaseSlopeNormal(iEdge)) - hydropotentialSlopeNormal(iEdge) = min(0.0_RKIND, hydropotentialSlopeNormal(iEdge)) - else ! cell1 is the cell outside the hydro domain - hydropotentialBaseSlopeNormal(iEdge) = max(0.0_RKIND, hydropotentialBaseSlopeNormal(iEdge)) - hydropotentialSlopeNormal(iEdge) = max(0.0_RKIND, hydropotentialSlopeNormal(iEdge)) + if (li_mask_is_grounded_ice(cellMask(cell1))) then ! cell2 is the icefree cell - replace phi there with cell1 Phig + + hydropotentialBaseSlopeNormal(iEdge) = - waterPressure(cell1) / dcEdge(iEdge) + hydropotentialSlopeNormal(iEdge) = - (rho_water * gravity * waterThickness(cell1) + waterPressure(cell1)) / dcEdge(iEdge) + + else ! cell1 is the icefree cell - replace phi there with cell2 Phig + + hydropotentialBaseSlopeNormal(iEdge) = waterPressure(cell2) / dcEdge(iEdge) + hydropotentialSlopeNormal(iEdge) = (rho_water * gravity * waterThickness(cell2) + waterPressure(cell2)) / dcEdge(iEdge) + endif ! which cell is icefree endif ! if edge of grounded ice end do - ! At terrestrial margin, ignore the downslope bed topography gradient. Including it can lead to unrealistically large - ! hydropotential gradients and unstable channel growth. - ! We also want to do this at marine margins because otherwise the offshore topography can create a barrier to flow, - ! but that is unrealistic. - ! So for all boundaries of the hydro system where outflow is occuring, - ! the hydropotential at the margin should be determined by the geometry - ! at the edge of the cell in a 1-sided sense. + ! Disallow inflow from the marine margin by imposing a minimum outflowing hydropotential gradient at the grounding line. do iEdge = 1, nEdges - if ( (li_mask_is_margin(edgeMask(iEdge)) .and. li_mask_is_grounded_ice(edgeMask(iEdge))) .or. & - (hydroMarineMarginMask(iEdge)==1)) then + if ( hydroMarineMarginMask(iEdge) == 1) then cell1 = cellsOnEdge(1, iEdge) cell2 = cellsOnEdge(2, iEdge) - if (li_mask_is_grounded_ice(cellMask(cell1))) then ! cell2 is the icefree cell - replace phi there with cell1 Phig - hydropotentialBaseSlopeNormal(iEdge) = (rho_water * gravity * bedTopography(cell1) + & - max(rhoo * gravity * (config_sea_level - bedTopography(cell1)), 0.0_RKIND) - & - hydropotentialBase(cell1)) / dcEdge(iEdge) - hydropotentialSlopeNormal(iEdge) = (rho_water * gravity * bedTopography(cell1) + & - max(rhoo * gravity * (config_sea_level - bedTopography(cell1)), 0.0_RKIND) - & - hydropotential(cell1)) / dcEdge(iEdge) - else ! cell1 is the icefree cell - replace phi there with cell2 Phig - hydropotentialBaseSlopeNormal(iEdge) = (hydropotentialBase(cell2) - & - ( rho_water * gravity * bedTopography(cell2) + & - max(rhoo * gravity * (config_sea_level - bedTopography(cell2)), 0.0_RKIND) ) ) / dcEdge(iEdge) - hydropotentialSlopeNormal(iEdge) = (hydropotential(cell2) - & - ( rho_water * gravity * bedTopography(cell2) + & - max(rhoo * gravity * (config_sea_level - bedTopography(cell2)), 0.0_RKIND) ) ) / dcEdge(iEdge) + if (li_mask_is_grounded_ice(cellMask(cell1))) then ! cell2 is the cell outside the hydro domain + + if (hydropotentialBaseSlopeNormal(iEdge) > -MIN_PHISLOPE_GL) then + hydropotentialBaseSlopeNormal(iEdge) = -MIN_PHISLOPE_GL + endif + if (hydropotentialSlopeNormal(iEdge) > -MIN_PHISLOPE_GL) then + hydropotentialSlopeNormal(iEdge) = -MIN_PHISLOPE_GL + endif + + else ! cell1 is the cell outside the hydro domain + + if (hydropotentialBaseSlopeNormal(iEdge) < MIN_PHISLOPE_GL) then + hydropotentialBaseSlopeNormal(iEdge) = MIN_PHISLOPE_GL + endif + if (hydropotentialSlopeNormal(iEdge) < MIN_PHISLOPE_GL) then + hydropotentialSlopeNormal(iEdge) = MIN_PHISLOPE_GL + endif + endif ! which cell is icefree endif ! if edge of grounded ice end do - ! zero gradients at edges that are marked as no flux. These should be applied at boundaries of the mesh. + ! zero gradients along zero flux boundaries do iEdge = 1, nEdges if (waterFluxMask(iEdge) == 2) then hydropotentialBaseSlopeNormal(iEdge) = 0.0_RKIND @@ -1516,7 +1565,7 @@ subroutine calc_pressure(block, err) real (kind=RKIND), dimension(:), pointer :: waterPressureOld real (kind=RKIND), dimension(:), pointer :: waterPressureTendency real (kind=RKIND), dimension(:), pointer :: waterThickness - real (kind=RKIND), dimension(:), pointer :: effectivePressure + real (kind=RKIND), dimension(:), pointer :: effectivePressureSGH real (kind=RKIND), dimension(:), pointer :: zeroOrderSum real (kind=RKIND), dimension(:), pointer :: closingRate real (kind=RKIND), dimension(:), pointer :: openingRate @@ -1575,7 +1624,7 @@ subroutine calc_pressure(block, err) call mpas_pool_get_array(meshPool, 'nEdgesOnCell', nEdgesOnCell) call mpas_pool_get_array(meshPool, 'edgesOnCell', edgesOnCell) - call mpas_pool_get_array(hydroPool, 'effectivePressure', effectivePressure) + call mpas_pool_get_array(hydroPool, 'effectivePressureSGH', effectivePressureSGH) call mpas_pool_get_array(hydroPool, 'waterPressure', waterPressure) call mpas_pool_get_array(hydroPool, 'waterPressureOld', waterPressureOld) call mpas_pool_get_array(hydroPool, 'waterPressureTendency', waterPressureTendency) @@ -1605,8 +1654,8 @@ subroutine calc_pressure(block, err) openingRate = max(0.0_RKIND, openingRate) closingRate(:) = creepCoeff * flowParamA(nVertLevels, :) * & - effectivePressure(:)**3 * waterThickness(:) -! closingRate(:) = waterThickness(:) * effectivePressure(:) / 1.0e13_RKIND + effectivePressureSGH(:)**3 * waterThickness(:) +! closingRate(:) = waterThickness(:) * effectivePressureSGH(:) / 1.0e13_RKIND ! ! Hewitt 2011 creep closure form. Denominator is ice viscosity zeroOrderSum = closingRate - openingRate + (basalMeltInput + externalWaterInput) / rho_water - & @@ -1617,22 +1666,23 @@ subroutine calc_pressure(block, err) select case (trim(config_SGH_pressure_calc)) case ('cavity') - where (li_mask_is_floating_ice(cellMask)) - waterPressure = rhoi * gravity * iceThicknessHydro - elsewhere (.not. li_mask_is_ice(cellMask)) - waterPressure = 0.0_RKIND - elsewhere + where (li_mask_is_grounded_ice(cellMask)) waterPressure = (zeroOrderSum - divergence - divergenceChannel - channelAreaChangeCell) * & - rho_water * gravity * deltatSGH / porosity + waterPressureOld + rho_water * gravity * deltatSGH / porosity + waterPressureOld + elsewhere ((.not. (li_mask_is_grounded_ice(cellMask))) .and. (bedTopography > config_sea_level)) + waterPressure = 0.0_RKIND + elsewhere ! should evaluate to ((.not. (li_mask_is_grounded_ice(cellMask))) .and. (bedTopography <= config_sea_level)) + waterPressure = gravity * rhoo * (config_sea_level - bedTopography) end where case ('overburden') - where (li_mask_is_floating_ice(cellMask)) + + where (li_mask_is_grounded_ice(cellMask)) waterPressure = rhoi * gravity * iceThicknessHydro - elsewhere (.not. li_mask_is_ice(cellMask)) + elsewhere ((.not. (li_mask_is_grounded_ice(cellMask))) .and. (bedTopography > config_sea_level)) waterPressure = 0.0_RKIND - elsewhere - waterPressure = rhoi * gravity * iceThicknessHydro + elsewhere ! should evaluate to ((.not. (li_mask_is_grounded_ice(cellMask))) .and. (bedTopography <= config_sea_level)) + waterPressure = gravity * rhoo * (config_sea_level - bedTopography) end where case default @@ -1641,27 +1691,23 @@ subroutine calc_pressure(block, err) end select waterPressure = max(0.0_RKIND, waterPressure) - waterPressure = min(waterPressure, rhoi * gravity * iceThicknessHydro) + where (li_mask_is_grounded_ice(cellMask)) + waterPressure = min(waterPressure, rhoi * gravity * iceThicknessHydro) + end where do iCell = 1, nCells - if ( li_mask_is_floating_ice(cellMask(iCell)) .or. & - ((.not. li_mask_is_ice(cellMask(iCell))) .and. (bedTopography(iCell) < config_sea_level) ) ) then - ! set pressure correctly under floating ice and open ocean - waterPressure(iCell) = rhoo * gravity * (config_sea_level - bedTopography(iCell)) - else - onMarineMargin = .false. - do iEdge = 1, nEdgesOnCell(iCell) - if (hydroMarineMarginMask(edgesOnCell(iEdge, iCell)) == 1) then - onMarineMargin = .true. - exit - endif - enddo - if (onMarineMargin) then - ! At marine margin, don't let water pressure fall below ocean pressure - ! TODO: Not sure if this should include the water layer thickness term. Leaving it off. - if (waterPressure(iCell) < rhoo * gravity * (config_sea_level - bedTopography(iCell))) then - waterPressure(iCell) = rhoo * gravity * (config_sea_level - bedTopography(iCell)) - endif + onMarineMargin = .false. + do iEdge = 1, nEdgesOnCell(iCell) + if (hydroMarineMarginMask(edgesOnCell(iEdge, iCell)) == 1) then + onMarineMargin = .true. + exit + endif + enddo + if (onMarineMargin) then + ! At marine margin, don't let water pressure fall below ocean pressure + ! TODO: Not sure if this should include the water layer thickness term. Leaving it off. + if (waterPressure(iCell) < rho_water * gravity * (config_sea_level - bedTopography(iCell))) then + waterPressure(iCell) = rho_water * gravity * (config_sea_level - bedTopography(iCell)) endif endif enddo @@ -1712,7 +1758,7 @@ subroutine calc_pressure_diag_vars(block, err) real (kind=RKIND), dimension(:), pointer :: hydropotentialBase real (kind=RKIND), dimension(:), pointer :: waterThickness real (kind=RKIND), dimension(:), pointer :: hydropotential - real (kind=RKIND), dimension(:), pointer :: effectivePressure + real (kind=RKIND), dimension(:), pointer :: effectivePressureSGH real (kind=RKIND), dimension(:), pointer :: iceThicknessHydro integer, dimension(:), pointer :: cellMask real (kind=RKIND), pointer :: config_sea_level @@ -1727,7 +1773,7 @@ subroutine calc_pressure_diag_vars(block, err) call mpas_pool_get_config(liConfigs, 'config_sea_level', config_sea_level) call mpas_pool_get_config(liConfigs, 'config_ocean_density', rhoo) - call mpas_pool_get_array(hydroPool, 'effectivePressure', effectivePressure) + call mpas_pool_get_array(hydroPool, 'effectivePressureSGH', effectivePressureSGH) call mpas_pool_get_array(hydroPool, 'waterPressure', waterPressure) call mpas_pool_get_array(geometryPool, 'bedTopography', bedTopography) call mpas_pool_get_array(hydroPool, 'hydropotentialBase', hydropotentialBase) @@ -1736,17 +1782,18 @@ subroutine calc_pressure_diag_vars(block, err) call mpas_pool_get_array(hydroPool, 'iceThicknessHydro', iceThicknessHydro) call mpas_pool_get_array(geometryPool, 'cellMask', cellMask) - effectivePressure = rhoi * gravity * iceThicknessHydro - waterPressure + effectivePressureSGH = rhoi * gravity * iceThicknessHydro - waterPressure ! < this should evalute to 0 for floating ice if Pw set correctly there. - where (.not. li_mask_is_grounded_ice(cellmask)) - effectivePressure = 0.0_RKIND ! zero effective pressure where no ice to avoid confusion + where (.not. (li_mask_is_grounded_ice(cellMask))) + effectivePressureSGH = 0.0_RKIND ! zero effective pressure where no ice to avoid confusion end where - + hydropotentialBase = rho_water * gravity * bedTopography + waterPressure - ! This is still correct under ice shelves/open ocean because waterPressure has been set appropriately there already. - ! Note this leads to a nonuniform hydropotential at sea level that is a function of the ocean depth. - ! That is what we want because we use this as a boundary condition on the subglacial system, - ! and we want the subglacial system to feel the pressure of the ocean column at its edge. + where ((.not. li_mask_is_grounded_ice(cellMask)) .and. (bedTopography <= config_sea_level)) + hydropotentialBase = 0.0_RKIND !zero hydropotential in ocean + elsewhere ((.not. li_mask_is_grounded_ice(cellMask)) .and. (bedTopography > config_sea_level)) + hydropotentialBase = rho_water * gravity * bedTopography ! for completeness, but won't matter with one-side slope calculations on terrestrial boundaries + end where ! hydropotential with water thickness hydropotential = hydropotentialBase + rho_water * gravity * waterThickness @@ -1812,8 +1859,9 @@ subroutine update_channel(block, err) real (kind=RKIND), dimension(:), pointer :: channelChangeRate real (kind=RKIND), dimension(:), pointer :: flowParamAChannel real (kind=RKIND), dimension(:), pointer :: channelEffectivePressure - real (kind=RKIND), dimension(:), pointer :: effectivePressure + real (kind=RKIND), dimension(:), pointer :: effectivePressureSGH real (kind=RKIND), dimension(:), pointer :: channelDiffusivity + real (kind=RKIND), dimension(:), pointer :: waterThicknessEdgeUpwind integer, dimension(:), pointer :: waterFluxMask integer, dimension(:), pointer :: hydroMarineMarginMask integer, dimension(:), pointer :: edgeMask @@ -1855,10 +1903,11 @@ subroutine update_channel(block, err) call mpas_pool_get_array(hydroPool, 'channelEffectivePressure', channelEffectivePressure) call mpas_pool_get_array(meshPool, 'cellsOnEdge', cellsOnEdge) call mpas_pool_get_array(velocityPool, 'flowParamA', flowParamA) - call mpas_pool_get_array(hydroPool, 'effectivePressure', effectivePressure) + call mpas_pool_get_array(hydroPool, 'effectivePressureSGH', effectivePressureSGH) call mpas_pool_get_array(hydroPool, 'waterFluxMask', waterFluxMask) call mpas_pool_get_array(hydroPool, 'hydroMarineMarginMask', hydroMarineMarginMask) call mpas_pool_get_array(hydroPool, 'channelDiffusivity', channelDiffusivity) + call mpas_pool_get_array(hydroPool, 'waterThicknessEdgeUpwind', waterThicknessEdgeUpwind) call mpas_pool_get_array(geometryPool, 'edgeMask', edgeMask) ! Calculate terms needed for opening (melt) rate @@ -1881,9 +1930,8 @@ subroutine update_channel(block, err) channelDischarge = 0.0_RKIND end where - ! Disable channels from forming if there is no sheet flux - ! TODO: Make a function of sheet dissipation threshold? - where (abs(waterFlux) <= 1e-10_RKIND) + ! Similar to waterFlux, shut off channelDischarge if no water upstream. Prevents self-sustaining channels in the absence of distributed water + where (waterThicknessEdgeUpwind == 0.0_RKIND) channelArea = 0.0_RKIND channelDischarge = 0.0_RKIND end where @@ -1919,7 +1967,7 @@ subroutine update_channel(block, err) ! Not sure if these ought to be upwind average, but using centered flowParamAChannel(iEdge) = 0.5_RKIND * (flowParamA(nVertLevels, cell1) + flowParamA(nVertLevels, cell2) ) - channelEffectivePressure(iEdge) = 0.5_RKIND * (effectivePressure(cell1) + effectivePressure(cell2)) + channelEffectivePressure(iEdge) = 0.5_RKIND * (effectivePressureSGH(cell1) + effectivePressureSGH(cell2)) end do channelClosingRate(:) = creep_coeff * channelArea(:) * flowParamAChannel(:) * channelEffectivePressure(:)**3 diff --git a/components/mpas-albany-landice/src/mode_forward/mpas_li_time_integration_fe_rk.F b/components/mpas-albany-landice/src/mode_forward/mpas_li_time_integration_fe_rk.F index e4c58cc42b11..e6051a3ec753 100644 --- a/components/mpas-albany-landice/src/mode_forward/mpas_li_time_integration_fe_rk.F +++ b/components/mpas-albany-landice/src/mode_forward/mpas_li_time_integration_fe_rk.F @@ -39,6 +39,7 @@ module li_time_integration_fe_rk use li_constants use li_mesh use li_mask + use li_tracer_advection_fct_shared implicit none private @@ -87,6 +88,7 @@ subroutine li_time_integrator_forwardeuler_rungekutta(domain, err) use li_bedtopo use li_mask use li_advection, only: li_grounded_to_floating + use li_ocean_extrap !----------------------------------------------------------------- ! input variables @@ -114,7 +116,8 @@ subroutine li_time_integrator_forwardeuler_rungekutta(domain, err) logical, pointer :: config_calculate_damage logical, pointer :: config_finalize_damage_after_advection logical, pointer :: config_update_velocity_before_calving - character (len=StrKIND), pointer :: config_thickness_advection + character (len=StrKIND), pointer :: config_thickness_advection, & + config_tracer_advection character (len=StrKIND), pointer :: config_thermal_solver character (len=StrKIND), pointer :: config_time_integration integer, pointer :: config_rk_order, config_rk3_stages @@ -166,9 +169,10 @@ subroutine li_time_integrator_forwardeuler_rungekutta(domain, err) call mpas_pool_get_config(liConfigs, 'config_restore_calving_front', config_restore_calving_front) call mpas_pool_get_config(liConfigs, 'config_restore_calving_front_prevent_retreat', config_restore_calving_front_prevent_retreat) - call mpas_pool_get_config(liConfigs, 'config_calculate_damage',config_calculate_damage) + call mpas_pool_get_config(liConfigs, 'config_calculate_damage', config_calculate_damage) call mpas_pool_get_config(liConfigs, 'config_finalize_damage_after_advection', config_finalize_damage_after_advection) call mpas_pool_get_config(liConfigs, 'config_thickness_advection', config_thickness_advection) + call mpas_pool_get_config(liConfigs, 'config_tracer_advection', config_tracer_advection) call mpas_pool_get_config(liConfigs, 'config_thermal_solver', config_thermal_solver) call mpas_pool_get_config(liConfigs, 'config_rk_order', config_rk_order) call mpas_pool_get_config(liConfigs, 'config_rk3_stages', config_rk3_stages) @@ -241,14 +245,18 @@ subroutine li_time_integrator_forwardeuler_rungekutta(domain, err) call mpas_timer_stop("advancing clock") !TODO: Determine whether grounded melting should in fact be called first +! === Ocean forcing extrapolation into ice-shelf cavities =========== + call mpas_timer_start("ocean forcing extrapolation") + call li_ocean_extrap_solve(domain, err_tmp) + err = ior(err, err_tmp) + call mpas_timer_stop("ocean forcing extrapolation") + ! === Face melting for grounded ice =========== call mpas_timer_start("face melting for grounded ice") call li_face_melt_grounded_ice(domain, err_tmp) err = ior(err, err_tmp) call mpas_timer_stop("face melting for grounded ice") -! *** TODO: Should basal melt rate calculation and column physics go inside RK loop? *** - ! === Basal melting for floating ice =========== call mpas_timer_start("basal melting for floating ice") call li_basal_melt_floating_ice(domain, err_tmp) @@ -281,6 +289,17 @@ subroutine li_time_integrator_forwardeuler_rungekutta(domain, err) enddo deltatFull = deltat ! Save deltat in order to reset it at end of RK loop + if ( (trim(config_tracer_advection) == 'fct') .or. & + (trim(config_thickness_advection) == 'fct') ) then + call li_tracer_advection_fct_shared_init(geometryPool, err_tmp) + if (err_tmp /= 0) then + err = 1 + call mpas_log_write( & + 'Error encountered during fct tracer advection shared init', & + MPAS_LOG_ERR) + endif + endif + ! Set RK weights based on desired time integration method. Note ! that rkSubstepWeights are used to update at each sub-step, and ! are thus offset from the typical writing of the coefficients @@ -350,7 +369,13 @@ subroutine li_time_integrator_forwardeuler_rungekutta(domain, err) // ' is not supported with config_rk_order = $i', & intArgs=(/config_rk_order/), messageType=MPAS_LOG_ERR) return - endif + endif + + ! Calculate masks prior to RK loop, but do not update masks within the loop + ! to preserve the accuracy of time integration. + call li_calculate_mask(meshPool, velocityPool, geometryPool, err_tmp) + err = ior(err, err_tmp) + ! *** Start RK loop *** do rkStage = 1, nRKstages call mpas_log_write('beginning rk stage $i of $i', & @@ -387,9 +412,8 @@ subroutine li_time_integrator_forwardeuler_rungekutta(domain, err) layerThickness(:,:) = rkSSPweights(rkStage) * layerThicknessPrev(:,:) + & (1.0_RKIND - rkSSPweights(rkStage)) * layerThickness(:,:) thickness = sum(layerThickness, 1) - ! Calculate masks after updating thickness - call li_calculate_mask(meshPool, velocityPool, geometryPool, err_tmp) - err = ior(err, err_tmp) + ! Do not calculate masks after updating thickness! We need to keep masks + ! constant for now to preserve accuracy of time integration if (trim(config_thermal_solver) .ne. 'none') then do iCell = 1, nCells @@ -521,6 +545,17 @@ subroutine li_time_integrator_forwardeuler_rungekutta(domain, err) fluxAcrossGroundingLine(:) = fluxAcrossGroundingLineAccum(:) fluxAcrossGroundingLineOnCells(:) = fluxAcrossGroundingLineOnCellsAccum(:) + ! Deallocate arrays for fct + if ( (trim(config_thickness_advection) .eq. 'fct') .or. & + (trim(config_tracer_advection) .eq. 'fct') ) then + deallocate( nAdvCellsForEdge, & + advCellsForEdge, & + advCoefs, & + advCoefs3rd, & + advMaskHighOrder, & + advMask2ndOrder) + endif + ! Reset time step to full length after RK loop deltat = deltatFull @@ -1010,6 +1045,11 @@ subroutine advection_solver(domain, err) real (kind=RKIND), dimension(:), pointer :: thicknessNew real (kind=RKIND), dimension(:), pointer :: thickness real (kind=RKIND), dimension(:,:), pointer :: layerThickness + real (kind=RKIND), dimension(:,:), pointer :: normalVelocity + real (kind=RKIND), dimension(:,:), pointer :: layerNormalVelocity + + integer, dimension(:), pointer :: edgeMask + real (kind=RKIND) :: allowableDtACFL real (kind=RKIND), dimension(:,:), pointer :: temperature real (kind=RKIND), dimension(:,:), pointer :: waterFrac @@ -1020,6 +1060,7 @@ subroutine advection_solver(domain, err) character (len=StrKIND), pointer :: config_thickness_advection logical, pointer :: config_restore_thickness_after_advection character (len=StrKIND), pointer :: config_tracer_advection + character (len=StrKIND), pointer :: config_time_integration logical, pointer :: config_print_thickness_advection_info @@ -1033,6 +1074,7 @@ subroutine advection_solver(domain, err) call mpas_pool_get_config(liConfigs, 'config_thickness_advection', config_thickness_advection) call mpas_pool_get_config(liConfigs, 'config_tracer_advection', config_tracer_advection) + call mpas_pool_get_config(liConfigs, 'config_time_integration', config_time_integration) call mpas_pool_get_config(liConfigs, 'config_print_thickness_advection_info', config_print_thickness_advection_info) call mpas_pool_get_config(liConfigs, 'config_restore_thickness_after_advection', config_restore_thickness_after_advection) @@ -1059,7 +1101,20 @@ subroutine advection_solver(domain, err) call mpas_pool_get_subpool(block % structs, 'mesh', meshPool) call mpas_pool_get_subpool(block % structs, 'velocity', velocityPool) call mpas_pool_get_subpool(block % structs, 'geometry', geometryPool) - + if (trim(config_time_integration) == "runge_kutta") then + call mpas_pool_get_array(velocityPool, 'normalVelocity', normalVelocity) + call mpas_pool_get_array(geometryPool, 'edgeMask', edgeMask) + call mpas_pool_get_array(velocityPool, 'layerNormalVelocity', layerNormalVelocity) + + call li_layer_normal_velocity( & + meshPool, & + normalVelocity, & + edgeMask, & + layerNormalVelocity, & + allowableDtACFL, & + err_tmp) + err = ior(err,err_tmp) + endif call calculate_layerThicknessEdge(meshPool, geometryPool, velocityPool, err_tmp) err = ior(err,err_tmp) diff --git a/components/mpas-framework/Makefile b/components/mpas-framework/Makefile index 1f9b00844e72..f92606987bbb 100644 --- a/components/mpas-framework/Makefile +++ b/components/mpas-framework/Makefile @@ -396,11 +396,11 @@ gnu-cray: "FFLAGS_OPT = -O3 -m64 -ffree-line-length-none -fconvert=big-endian -ffree-form -ffpe-summary=none $${EXTRA_FFLAGS}" \ "CFLAGS_OPT = -O3 -m64" \ "CXXFLAGS_OPT = -O3 -m64" \ - "LDFLAGS_OPT = -O3 -m64" \ + "LDFLAGS_OPT = -O3 -m64 $(GNU_CRAY_LDFLAGS)" \ "FFLAGS_DEBUG = -g -m64 -ffree-line-length-none -fconvert=big-endian -ffree-form -fbounds-check -fbacktrace -ffpe-trap=invalid,zero,overflow -ffpe-summary=none $${EXTRA_FFLAGS}" \ "CFLAGS_DEBUG = -g -m64" \ "CXXFLAGS_DEBUG = -g -m64" \ - "LDFLAGS_DEBUG = -g -m64" \ + "LDFLAGS_DEBUG = -g -m64 $(GNU_CRAY_LDFLAGS)" \ "FFLAGS_OMP = -fopenmp" \ "CFLAGS_OMP = -fopenmp" \ "BUILD_TARGET = $(@)" \ diff --git a/components/mpas-framework/src/build_core.cmake b/components/mpas-framework/src/build_core.cmake index 23c1935a8abb..a2c361e069e5 100644 --- a/components/mpas-framework/src/build_core.cmake +++ b/components/mpas-framework/src/build_core.cmake @@ -62,6 +62,13 @@ function(build_core CORE) endforeach() endif() + # Disable optimizations on some files that would take too long to compile, expect these to all be fortran files + foreach (SOURCE_FILE IN LISTS NOOPT_FILES) + get_filename_component(SOURCE_EXT ${SOURCE_FILE} EXT) + string(REPLACE "${SOURCE_EXT}" ".f90" SOURCE_F90 ${SOURCE_FILE}) + e3sm_deoptimize_file(${CMAKE_BINARY_DIR}/${SOURCE_F90}) + endforeach() + genf90_targets("${RAW_SOURCES}" "${INCLUDES}" "${CPPDEFS}" "${NO_PREPROCESS}" "${INC_DIR}") target_sources(${COMPONENT} PRIVATE ${SOURCES}) diff --git a/components/mpas-framework/src/framework/mpas_forcing.F b/components/mpas-framework/src/framework/mpas_forcing.F index 950b103604c7..95de2ca61846 100644 --- a/components/mpas-framework/src/framework/mpas_forcing.F +++ b/components/mpas-framework/src/framework/mpas_forcing.F @@ -35,7 +35,8 @@ module mpas_forcing mpas_forcing_init_field_data, & mpas_forcing_get_forcing, & mpas_forcing_get_forcing_time, & - mpas_forcing_write_restart_times + mpas_forcing_write_restart_times, & + mpas_advance_forcing_clock contains @@ -1193,7 +1194,7 @@ subroutine mpas_forcing_get_forcing(&!{{{ if (trim(forcingGroup % forcingGroupName) == trim(forcingGroupName)) then ! advance the forcing time - call advance_forcing_clock(forcingGroup, dt) + call mpas_advance_forcing_clock(forcingGroup, dt) ! cycle the forcing clock if (forcingGroup % forcingCycleUse) then @@ -1230,7 +1231,7 @@ end subroutine mpas_forcing_get_forcing!}}} !||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| ! -! advance_forcing_clock +! mpas_advance_forcing_clock ! !> \brief set the forcing clock !> \author Adrian K. Turner, LANL @@ -1241,7 +1242,7 @@ end subroutine mpas_forcing_get_forcing!}}} ! !----------------------------------------------------------------------- - subroutine advance_forcing_clock(&!{{{ + subroutine mpas_advance_forcing_clock(&!{{{ forcingGroup, & dt) @@ -1256,9 +1257,10 @@ subroutine advance_forcing_clock(&!{{{ ! increment clock with timestep call mpas_set_timeInterval(timeStep, dt=dt) + !call mpas_log_write('time step from mpas_set_timeInterval: $i $i $i', intArgs=(/int(timeStep%ti%basetime%S), int(timeStep%ti%basetime%Sn), int(timeStep%ti%basetime%Sd) /)) call mpas_advance_clock(forcingGroup % forcingClock, timeStep) - end subroutine advance_forcing_clock!}}} + end subroutine mpas_advance_forcing_clock!}}} !||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| ! @@ -1597,8 +1599,10 @@ subroutine get_interpolants_linear(interpolants, forcingStream, currentTime)!{{{ call mpas_get_timeInterval(diff1, forcingStream % forcingTimes(1), dt=diffr1) call mpas_get_timeInterval(diff2, currentTime, dt=diffr2) + !call mpas_log_write('diffr2 $r, diffr, $r', realArgs=(/ diffr2, diffr /)) interpolants(1) = diffr2 / diffr interpolants(2) = 1.0_RKIND - interpolants(1) !diffr1 / diffr + end subroutine get_interpolants_linear!}}} @@ -1612,7 +1616,7 @@ end subroutine get_interpolants_linear!}}} !> \details !> Given the current time and forcing times calculate the correct !> interpolants with piecewise constant interpolation -! + !----------------------------------------------------------------------- subroutine get_interpolants_constant(interpolants, forcingStream, currentTime)!{{{ diff --git a/components/mpas-framework/src/framework/mpas_timekeeping.F b/components/mpas-framework/src/framework/mpas_timekeeping.F index 37a24d77358d..57d9651335a4 100644 --- a/components/mpas-framework/src/framework/mpas_timekeeping.F +++ b/components/mpas-framework/src/framework/mpas_timekeeping.F @@ -1427,6 +1427,8 @@ subroutine mpas_set_timeInterval(interval, YY, MM, DD, H, M, S, S_n, S_d, S_i8, timeString_ = timeString end if + !call mpas_log_write('timeString_ '//trim(timeString_)) + numerator = 0 denominator = 1 @@ -1519,6 +1521,12 @@ subroutine mpas_set_timeInterval(interval, YY, MM, DD, H, M, S, S_n, S_d, S_i8, return end if + if (sec < 0) then + numerator = -numerator + end if + + !call mpas_log_write('sec $i, num $i, denom $i', intArgs=(/int(sec),numerator,denominator/)) + call ESMF_TimeIntervalSet(interval % ti, YY=year, MM=month, D=day, H=hour, M=min, S_i8=sec, Sn=numerator, Sd=denominator, rc=ierr) else diff --git a/components/mpas-ocean/bld/build-namelist b/components/mpas-ocean/bld/build-namelist index 460b9ecda428..f2050753b5a8 100755 --- a/components/mpas-ocean/bld/build-namelist +++ b/components/mpas-ocean/bld/build-namelist @@ -725,12 +725,25 @@ add_default($nl, 'config_sgr_salinity_prescribed'); # Namelist group: coupling # ############################ -add_default($nl, 'config_remove_ais_river_runoff'); -if (($OCN_ICEBERG eq 'true') && ($OCN_FORCING eq 'active_atm')) { +# When we have a data representation of icebergs, remove ice runoff +if ($OCN_ICEBERG eq 'true') { add_default($nl, 'config_remove_ais_ice_runoff', 'val'=>".true."); } else { add_default($nl, 'config_remove_ais_ice_runoff', 'val'=>".false."); } +# Assume river runoff corresponds to ISMF in Southern Ocean only for G-cases +# (true for JRA). When atm active, we assume liquid runoff corresonds to precip +# or snow melt so we do not remove it. In either case, the energy for melting +# doesn't come from the ocean. +if (($OCN_ISMF ne 'none') && ($OCN_FORCING ne 'active_atm')) { + add_default($nl, 'config_remove_ais_river_runoff', 'val'=>".true."); +} else { + add_default($nl, 'config_remove_ais_river_runoff', 'val'=>".false."); +} +add_default($nl, 'config_scale_dismf_by_removed_ice_runoff'); +add_default($nl, 'config_ais_ice_runoff_history_days'); +add_default($nl, 'config_glc_thermal_forcing_coupling_mode'); +add_default($nl, 'config_2d_thermal_forcing_depth'); ###################################### # Namelist group: shortwaveRadiation # @@ -780,7 +793,6 @@ add_default($nl, 'config_self_attraction_and_loading_beta'); add_default($nl, 'config_use_frazil_ice_formation'); add_default($nl, 'config_frazil_in_open_ocean'); -add_default($nl, 'config_frazil_under_land_ice'); add_default($nl, 'config_frazil_heat_of_fusion'); add_default($nl, 'config_frazil_ice_density'); add_default($nl, 'config_frazil_fractional_thickness_limit'); @@ -790,18 +802,32 @@ add_default($nl, 'config_frazil_sea_ice_reference_salinity'); add_default($nl, 'config_frazil_maximum_freezing_temperature'); add_default($nl, 'config_frazil_use_surface_pressure'); +if ($OCN_ISMF eq 'coupled') { + add_default($nl, 'config_frazil_under_land_ice', 'val'=>".true."); +} elsif ($OCN_ISMF eq 'internal') { + add_default($nl, 'config_frazil_under_land_ice', 'val'=>".true."); +} elsif ($OCN_ISMF eq 'data') { + add_default($nl, 'config_frazil_under_land_ice', 'val'=>".false."); +} else { + add_default($nl, 'config_frazil_under_land_ice'); +} + ################################### # Namelist group: land_ice_fluxes # ################################### if ($OCN_ISMF eq 'coupled') { add_default($nl, 'config_land_ice_flux_mode', 'val'=>"coupled"); + add_default($nl, 'config_frazil_under_land_ice', 'val'=>".true."); } elsif ($OCN_ISMF eq 'internal') { add_default($nl, 'config_land_ice_flux_mode', 'val'=>"standalone"); + add_default($nl, 'config_frazil_under_land_ice', 'val'=>".true."); } elsif ($OCN_ISMF eq 'data') { add_default($nl, 'config_land_ice_flux_mode', 'val'=>"data"); + add_default($nl, 'config_frazil_under_land_ice', 'val'=>".false."); } else { add_default($nl, 'config_land_ice_flux_mode'); + add_default($nl, 'config_frazil_under_land_ice'); } if ($OCN_TIDAL_MIXING eq 'true') { add_default($nl, 'config_land_ice_flux_tidal_Jourdain_alpha', 'val'=>"0.777"); diff --git a/components/mpas-ocean/bld/build-namelist-section b/components/mpas-ocean/bld/build-namelist-section index c5dad5d935a9..74c699d878b4 100644 --- a/components/mpas-ocean/bld/build-namelist-section +++ b/components/mpas-ocean/bld/build-namelist-section @@ -239,6 +239,10 @@ add_default($nl, 'config_sgr_salinity_prescribed'); add_default($nl, 'config_remove_ais_river_runoff'); add_default($nl, 'config_remove_ais_ice_runoff'); +add_default($nl, 'config_scale_dismf_by_removed_ice_runoff'); +add_default($nl, 'config_ais_ice_runoff_history_days'); +add_default($nl, 'config_glc_thermal_forcing_coupling_mode'); +add_default($nl, 'config_2d_thermal_forcing_depth'); ###################################### # Namelist group: shortwaveRadiation # diff --git a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml index fd2785616655..73d4eeecd526 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml @@ -55,6 +55,7 @@ '00:02:00' '00:01:00' '00:05:00' +'00:10:00' 'split_explicit_ab2' 2 @@ -85,6 +86,7 @@ .true. .true. .true. +.true. -1.0 .false. 30.0e3 @@ -107,6 +109,7 @@ .true. .true. .true. +.true. 10.0 1000.0 1000.0 @@ -123,6 +126,7 @@ 77.0 38.5 100.0 +462.0 .false. 10.0 @@ -154,6 +158,7 @@ 5.46e07 6.83e06 3.2e09 +1.18e10 1.0 .false. 0.0 @@ -191,6 +196,7 @@ 'RossbyRadius' 'RossbyRadius' 'RossbyRadius' +'RossbyRadius' 20e3 30e3 30e3 @@ -222,6 +228,7 @@ 'N2_dependent' 'N2_dependent' 'N2_dependent' +'N2_dependent' 900.0 600.0 600.0 @@ -236,6 +243,7 @@ 600.0 600.0 600.0 +600.0 0.3 'constant' 300.0 @@ -249,6 +257,7 @@ 1.0 1.0 1.0 +1.0 0.13 1000.0 200.0 @@ -264,6 +273,7 @@ 'RossbyRadius' 'RossbyRadius' 'RossbyRadius' +'RossbyRadius' 20e3 30e3 30e3 @@ -354,6 +364,10 @@ .false. .false. +.false. +731 +'off' +300.0 'jerlov' @@ -391,7 +405,7 @@ .true. .true. -.true. +.false. 3.337e5 1000.0 0.1 @@ -416,6 +430,7 @@ 'pressure_only' 'pressure_only' 'pressure_only' +'pressure_only' 'Jenkins' .false. 10.0 @@ -435,6 +450,7 @@ 4.48e-3 4.48e-3 4.48e-3 +4.48e-3 1e-4 0.011 0.00295 @@ -448,6 +464,7 @@ 0.00295 0.00295 0.00295 +0.00295 3.1e-4 8.42e-5 8.42e-5 @@ -460,6 +477,7 @@ 8.42e-5 8.42e-5 8.42e-5 +8.42e-5 1.0 0.0 5e-2 @@ -492,6 +510,7 @@ 4.48e-3 4.48e-3 4.48e-3 +4.48e-3 1.0e-3 10.0 2.5e-3 @@ -579,6 +598,7 @@ '0000_00:00:02.5' '0000_00:00:01.25' '0000_00:00:05' +'0000_00:00:15' 2 .true. 2 @@ -614,6 +634,7 @@ .false. .true. +.false. .false. .false. .false. @@ -626,6 +647,7 @@ .false. .false. .false. +.false. .false. .false. .false. @@ -1120,6 +1142,7 @@ .false. +.true. .true. '0000-00-00_01:00:00' 'eddyProductVariablesOutput' @@ -1153,6 +1176,7 @@ .true. .true. .true. +.true. '0000-00-00_01:00:00' 'mocStreamfunctionOutput' .true. @@ -1232,37 +1256,13 @@ .false. -.true. -.true. -.true. -.true. -.true. -.true. -.true. -.true. -.true. +.true. 'dt' 'conservationCheckOutput' .false. -.true. -.true. -.true. -.true. -.true. -.true. -.true. -.true. -.true. +.true. .false. -.true. -.true. -.true. -.true. -.true. -.true. -.true. -.true. -.true. +.true. .true. 'conservationCheckRestart' diff --git a/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml index bea1e98d9de8..7a5e8bf1dda7 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml @@ -1271,6 +1271,38 @@ Valid values: .true. or .false. Default: Defined in namelist_defaults.xml
+ +Whether to scale data ice-shelf melt fluxes by the running mean of removed ice runoff. + +Valid values: .true. or .false. +Default: Defined in namelist_defaults.xml + + + +The number of days over for which the history of removed AIS runoff is stored. The default is 731 days (2 years + 1 day). + +Valid values: Any positive integer +Default: Defined in namelist_defaults.xml + + + +If and how MPAS-Ocean sends thermal forcing to GLC (MALI) in E3SM. This is used for ocean coupling with a melt parameterization for grounded marine ice-cliffs in MALI. This is primarily relevant to the Greenland Ice Sheet, but also relevant to the Antarctic Ice Sheet. 'none' means no coupling of thermal forcing. '2d' means thermal forcing at a prescribed depth is passed to GLC. That depth is controlled by 'config_2d_thermal_forcing_depth', and the resulting thermal forcing field is calculated in the field 'avgThermalForcingAtCritDepth'. + +Valid values: 'off', '2d' +Default: Defined in namelist_defaults.xml + + + +Depth at which to pass 2d thermal forcing to the coupler for use in the GLC component. Note that mapping files for this field must be created with a mask to exclude ocean grid cells shallower than this value and thus must be regenerated if this value is changed. + +Valid values: any non-negative value +Default: Defined in namelist_defaults.xml + + diff --git a/components/mpas-ocean/cime_config/SystemTests/mvko.py b/components/mpas-ocean/cime_config/SystemTests/mvko.py index 71477a4dfc12..90fa4b7a5539 100644 --- a/components/mpas-ocean/cime_config/SystemTests/mvko.py +++ b/components/mpas-ocean/cime_config/SystemTests/mvko.py @@ -18,6 +18,7 @@ import netCDF4 as nc import CIME.test_status +from CIME.status import append_testlog import CIME.utils from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.case.case_setup import case_setup @@ -394,4 +395,4 @@ def _compare_baseline(self): f" {viewing}" ) - CIME.utils.append_testlog(comments, self._orig_caseroot) + append_testlog(comments, self._orig_caseroot) diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index 8f5193c334be..778d268d5700 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -376,7 +376,7 @@ def buildnml(case, caseroot, compname): ic_prefix = 'mpaso.IcoswISC30E3r5.20231120+MARBL_ICfromOMIP_64levels' eco_forcing_file = 'ecoForcingSurfaceMonthly.IcoswISC30E3r5.20231215.nc' if ocn_ismf == 'data': - data_ismf_file = 'prescribed_ismf_paolo2023.IcoswISC30E3r5.20240227.nc' + data_ismf_file = 'prescribed_ismf_paolo2023.IcoswISC30E3r5.20240805.nc' if ocn_tidal_mixing == 'true': u_tidal_rms_file = 'velocityTidalRMS_CATS2008.IcoswISC30E3r5.20231120.nc' if ocn_sgr == 'data': @@ -406,6 +406,20 @@ def buildnml(case, caseroot, compname): if ocn_ismf == 'data': data_ismf_file = 'prescribed_ismf_paolo2023.RRSwISC6to18E3r5.20240327.nc' + elif ocn_grid == 'SOwISC12to30E3r3': + decomp_date = '20240829' + decomp_prefix = 'partitions/mpas-o.graph.info.' + restoring_file = 'sss.PHC2_monthlyClimatology.SOwISC12to30E3r3.20240829.nc' + analysis_mask_file = 'SOwISC12to30E3r3_mocBasinsAndTransects20210623.nc' + ic_date = '20240829' + ic_prefix = 'mpaso.SOwISC12to30E3r3' + if ocn_ic_mode == 'spunup': + ic_date = '20240829' + ic_prefix = 'mpaso.SOwISC12to30E3r3.rstFromG-chrysalis' + if ocn_ismf == 'data': + data_ismf_file = 'prescribed_ismf_paolo2023.SOwISC12to30E3r3.20241017.nc' + + #-------------------------------------------------------------------- # Set OCN_FORCING = datm_forced_restoring if restoring file is available #-------------------------------------------------------------------- @@ -948,6 +962,7 @@ def buildnml(case, caseroot, compname): lines.append(' ') lines.append(' ') lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') @@ -1156,7 +1171,7 @@ def buildnml(case, caseroot, compname): lines.append(' filename_interval="00-01-00_00:00:00"') lines.append(' reference_time="01-01-01_00:00:00"') lines.append(' output_interval="00-00-01_00:00:00"') - lines.append(' clobber_mode="truncate"') + lines.append(' clobber_mode="append"') lines.append(' packages="conservationCheckAMPKG">') lines.append('') lines.append('') @@ -1803,6 +1818,7 @@ def buildnml(case, caseroot, compname): lines.append(' packages="dataLandIceFluxesPKG">') lines.append(' ') lines.append(' ') + lines.append(' ') lines.append('') lines.append('') diff --git a/components/mpas-ocean/cime_config/config_component.xml b/components/mpas-ocean/cime_config/config_component.xml index 83b4501ee5e5..ee55bf415d92 100644 --- a/components/mpas-ocean/cime_config/config_component.xml +++ b/components/mpas-ocean/cime_config/config_component.xml @@ -107,6 +107,8 @@ constant none + none + none constant constant diagnostic diff --git a/components/mpas-ocean/cime_config/config_compsets.xml b/components/mpas-ocean/cime_config/config_compsets.xml index be36f0103472..62c87ddea3fd 100644 --- a/components/mpas-ocean/cime_config/config_compsets.xml +++ b/components/mpas-ocean/cime_config/config_compsets.xml @@ -34,17 +34,17 @@ GMPAS-NYF-PISMF - 2000_DATM%NYF_SLND_MPASSI_MPASO%PISMFDATMFORCED_DROF%NYFAIS45_SGLC_SWAV + 2000_DATM%NYF_SLND_MPASSI_MPASO%PISMFDATMFORCED_DROF%NYF_SGLC_SWAV GMPAS-NYF-PISMF-DSGR - 2000_DATM%NYF_SLND_MPASSI_MPASO%PISMFDATMFORCEDDSGR_DROF%NYFAIS45_SGLC_SWAV + 2000_DATM%NYF_SLND_MPASSI_MPASO%PISMFDATMFORCEDDSGR_DROF%NYF_SGLC_SWAV GMPAS-NYF-DISMF - 2000_DATM%NYF_SLND_MPASSI_MPASO%DISMFDATMFORCED_DROF%NYFAIS45_SGLC_SWAV + 2000_DATM%NYF_SLND_MPASSI_MPASO%DISMFDATMFORCED_DROF%NYF_SGLC_SWAV @@ -67,6 +67,11 @@ 2000_DATM%JRA-1p5_SLND_MPASSI_MPASO%DATMFORCED_DROF%JRA-1p5_SGLC_SWAV + + GMPAS-JRA1p5-2023 + 2000_DATM%JRA-1p5-2023_SLND_MPASSI_MPASO%DATMFORCED_DROF%JRA-1p5-2023_SGLC_SWAV + + GMPAS-OECO-JRA1p4 2000_DATM%JRA-1p4-2018_SLND_MPASSI_MPASO%OECODATMFORCED_DROF%JRA-1p4-2018_SGLC_SWAV @@ -89,27 +94,27 @@ GMPAS-JRA1p5-DIB-PISMF - 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCED_DROF%JRA-1p5-AIS0ROF_SGLC_SWAV + 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCED_DROF%JRA-1p5_SGLC_SWAV GMPAS-JRA1p5-DIB-PISMF-DSGR - 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCEDDSGR_DROF%JRA-1p5-AIS0ROF_SGLC_SWAV + 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCEDDSGR_DROF%JRA-1p5_SGLC_SWAV GMPAS-JRA1p5-DIB-PISMF-DSGR-TMIX - 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCEDDSGRTMIX_DROF%JRA-1p5-AIS0ROF_SGLC_SWAV + 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCEDDSGRTMIX_DROF%JRA-1p5_SGLC_SWAV GMPAS-JRA1p5-DIB-DISMF - 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBDISMFDATMFORCED_DROF%JRA-1p5-AIS0ROF_SGLC_SWAV + 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBDISMFDATMFORCED_DROF%JRA-1p5_SGLC_SWAV GMPAS-JRA1p5-DIB-PISMF-TMIX - 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCEDTMIX_DROF%JRA-1p5-AIS0ROF_SGLC_SWAV + 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCEDTMIX_DROF%JRA-1p5_SGLC_SWAV @@ -119,77 +124,77 @@ GMPAS-JRA1p4-PISMF - 2000_DATM%JRA-1p4-2018_SLND_MPASSI_MPASO%PISMFDATMFORCED_DROF%JRA-1p4-2018-AIS0LIQ_SGLC_SWAV + 2000_DATM%JRA-1p4-2018_SLND_MPASSI_MPASO%PISMFDATMFORCED_DROF%JRA-1p4-2018_SGLC_SWAV GMPAS-JRA1p4-DISMF - 2000_DATM%JRA-1p4-2018_SLND_MPASSI_MPASO%DISMFDATMFORCED_DROF%JRA-1p4-2018-AIS0LIQ_SGLC_SWAV + 2000_DATM%JRA-1p4-2018_SLND_MPASSI_MPASO%DISMFDATMFORCED_DROF%JRA-1p4-2018_SGLC_SWAV GMPAS-JRA1p4-DIB - 2000_DATM%JRA-1p4-2018_SLND_MPASSI%DIB_MPASO%IBDATMFORCED_DROF%JRA-1p4-2018-AIS0ICE_SGLC_SWAV + 2000_DATM%JRA-1p4-2018_SLND_MPASSI%DIB_MPASO%IBDATMFORCED_DROF%JRA-1p4-2018_SGLC_SWAV GMPAS-JRA1p4-DIB-PISMF - 2000_DATM%JRA-1p4-2018_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCED_DROF%JRA-1p4-2018-AIS0ROF_SGLC_SWAV + 2000_DATM%JRA-1p4-2018_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCED_DROF%JRA-1p4-2018_SGLC_SWAV GMPAS-JRA1p4-DIB-DISMF - 2000_DATM%JRA-1p4-2018_SLND_MPASSI%DIB_MPASO%IBDISMFDATMFORCED_DROF%JRA-1p4-2018-AIS0ROF_SGLC_SWAV + 2000_DATM%JRA-1p4-2018_SLND_MPASSI%DIB_MPASO%IBDISMFDATMFORCED_DROF%JRA-1p4-2018_SGLC_SWAV GMPAS-IAF-PISMF - 2000_DATM%IAF_SLND_MPASSI_MPASO%PISMFDATMFORCED_DROF%IAFAIS45_SGLC_SWAV + 2000_DATM%IAF_SLND_MPASSI_MPASO%PISMFDATMFORCED_DROF%IAF_SGLC_SWAV GMPAS-IAF-DISMF - 2000_DATM%IAF_SLND_MPASSI_MPASO%DISMFDATMFORCED_DROF%IAFAIS45_SGLC_SWAV + 2000_DATM%IAF_SLND_MPASSI_MPASO%DISMFDATMFORCED_DROF%IAF_SGLC_SWAV GMPAS-DIB-NYF - 2000_DATM%NYF_SLND_MPASSI%DIB_MPASO%IBDATMFORCED_DROF%NYFAIS55_SGLC_SWAV + 2000_DATM%NYF_SLND_MPASSI%DIB_MPASO%IBDATMFORCED_DROF%NYF_SGLC_SWAV GMPAS-DIB-NYF-PISMF - 2000_DATM%NYF_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCED_DROF%NYFAIS00_SGLC_SWAV + 2000_DATM%NYF_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCED_DROF%NYF_SGLC_SWAV GMPAS-DIB-NYF-DISMF - 2000_DATM%NYF_SLND_MPASSI%DIB_MPASO%IBDISMFDATMFORCED_DROF%NYFAIS00_SGLC_SWAV + 2000_DATM%NYF_SLND_MPASSI%DIB_MPASO%IBDISMFDATMFORCED_DROF%NYF_SGLC_SWAV GMPAS-DIB-IAF - 2000_DATM%IAF_SLND_MPASSI%DIB_MPASO%IBDATMFORCED_DROF%IAFAIS55_SGLC_SWAV + 2000_DATM%IAF_SLND_MPASSI%DIB_MPASO%IBDATMFORCED_DROF%IAF_SGLC_SWAV GMPAS-DIB-IAF-PISMF - 2000_DATM%IAF_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCED_DROF%IAFAIS00_SGLC_SWAV + 2000_DATM%IAF_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCED_DROF%IAF_SGLC_SWAV GMPAS-DIB-IAF-DISMF - 2000_DATM%IAF_SLND_MPASSI%DIB_MPASO%IBDISMFDATMFORCED_DROF%IAFAIS00_SGLC_SWAV + 2000_DATM%IAF_SLND_MPASSI%DIB_MPASO%IBDISMFDATMFORCED_DROF%IAF_SGLC_SWAV GMPAS-MALI-DIB-IAF-DISMF - 2000_DATM%IAF_SLND_MPASSI%DIB_MPASO%IBDISMFCOREFORCED_DROF%IAFAIS00_MALI%SIASTATIC_SWAV + 2000_DATM%IAF_SLND_MPASSI%DIB_MPASO%IBDISMFCOREFORCED_DROF%IAF_MALI%SIASTATIC_SWAV GMPAS-MALI-DIB-IAF-DISMF - 2000_DATM%IAF_SLND_MPASSI%DIB_MPASO%IBDISMFCOREFORCED_DROF%IAFAIS00_MALI%SIASTATIC_SWAV + 2000_DATM%IAF_SLND_MPASSI%DIB_MPASO%IBDISMFCOREFORCED_DROF%IAF_MALI%SIASTATIC_SWAV diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/README b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/README new file mode 100644 index 000000000000..fb85b185fd47 --- /dev/null +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/README @@ -0,0 +1,8 @@ +This testdef is used to test the conservation check analysis member, which +wasintroduced in MPAS-Ocean PR #4521 and has been made a stealth feature in +#6643. This test turns on the consevation check analysis member by setting: + + config_AM_conservationCheck_enable = .true. + +However, it should be noted that MPAS-Ocean history files are not currently +included in E3SM testing so non-BFB results will not be detected. diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands new file mode 100644 index 000000000000..764340d699f9 --- /dev/null +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands @@ -0,0 +1,5 @@ +# include mpas-ocean outputs in testing +sed -i 's/\(compname="mpaso" exclude_testing="true"\)/compname="mpaso"/' env_archive.xml +sed -i '/\(compname="mpaso"\)/,// s/hist/hist.am.conservationCheck\\..*\\.nc$/' env_archive.xml +sed -i 's/casename.mpaso.hist.am.globalStats.1976-01-01.nc/casename.mpaso.hist.am.conservationCheck.1976-01-01.nc/' env_archive.xml +sed -i '/casename.mpaso.hist.am.highFrequencyOutput.1976-01-01_00.00.00.nc/d' env_archive.xml diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/user_nl_mpaso b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/user_nl_mpaso new file mode 100644 index 000000000000..c57241a9fcf2 --- /dev/null +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/user_nl_mpaso @@ -0,0 +1 @@ + config_AM_conservationCheck_enable = .true. diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/ocn_glc_tf_coupling/README b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/ocn_glc_tf_coupling/README new file mode 100644 index 000000000000..bd801c44f8af --- /dev/null +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/ocn_glc_tf_coupling/README @@ -0,0 +1,12 @@ +This testdef is used to test a stealth feature that enables coupling between +OCN and GLC for Greenland, which passes ocean thermal forcing from OCN to GLC +and uses that in a parameterization for marine melting of grounded vertical +cliffs. + +It changes one mpaso namelist variable, + config_glc_thermal_forcing_coupling_mode +from its default value to '2d'. +This tests the ocn/glc TF coupling. + +It also specified that DATM forcing should be restricted to 1958. +This allows JRA1p5 forcing to be used without a large input data requirement. diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/ocn_glc_tf_coupling/shell_commands b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/ocn_glc_tf_coupling/shell_commands new file mode 100644 index 000000000000..1d43ad8c5baf --- /dev/null +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/ocn_glc_tf_coupling/shell_commands @@ -0,0 +1,4 @@ +./xmlchange DATM_CLMNCEP_YR_START=1958 +./xmlchange DATM_CLMNCEP_YR_END=1958 +./xmlchange DROF_STRM_YR_START=1958 +./xmlchange DROF_STRM_YR_END=1958 diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/ocn_glc_tf_coupling/user_nl_mpaso b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/ocn_glc_tf_coupling/user_nl_mpaso new file mode 100644 index 000000000000..0e1378620836 --- /dev/null +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/ocn_glc_tf_coupling/user_nl_mpaso @@ -0,0 +1 @@ +config_glc_thermal_forcing_coupling_mode = '2d' diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/scaled_dib_dismf/README b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/scaled_dib_dismf/README new file mode 100644 index 000000000000..f9f97d27de15 --- /dev/null +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/scaled_dib_dismf/README @@ -0,0 +1,8 @@ +This testdef is used to test a stealth feature in mpaso and mpassi introduced +by PR #6696. It simply changes sets the following namelist option in mpaso +to true + config_scale_dismf_by_removed_ice_runoff +and the following namelist option in mpassi to true + config_scale_dib_by_removed_ice_runoff +These flags only makes sense to use in a B-case runs with data icebergs (DIB) +and data ice-shelf melt fluxes (DISMF). diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/scaled_dib_dismf/user_nl_mpaso b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/scaled_dib_dismf/user_nl_mpaso new file mode 100644 index 000000000000..902d4418f5a0 --- /dev/null +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/scaled_dib_dismf/user_nl_mpaso @@ -0,0 +1 @@ + config_scale_dismf_by_removed_ice_runoff = .true. diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/scaled_dib_dismf/user_nl_mpassi b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/scaled_dib_dismf/user_nl_mpassi new file mode 100644 index 000000000000..8512001af932 --- /dev/null +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/scaled_dib_dismf/user_nl_mpassi @@ -0,0 +1 @@ + config_scale_dib_by_removed_ice_runoff = .true. diff --git a/components/mpas-ocean/driver/mpaso_cpl_indices.F b/components/mpas-ocean/driver/mpaso_cpl_indices.F index f099cf8ea46a..9802d526bc50 100644 --- a/components/mpas-ocean/driver/mpaso_cpl_indices.F +++ b/components/mpas-ocean/driver/mpaso_cpl_indices.F @@ -37,6 +37,7 @@ module mpaso_cpl_indices integer :: index_o2x_So_htv !ocean heat-transfer velocity integer :: index_o2x_So_stv !ocean salt-transfer velocity integer :: index_o2x_So_rhoeff !ocean effective density + integer :: index_o2x_So_tf2d !ocean thermal forcing at predefined critical depth ! ocn -> drv (BGC) @@ -208,6 +209,7 @@ subroutine mpaso_cpl_indices_set( ) index_o2x_So_htv = mct_avect_indexra(o2x,'So_htv') index_o2x_So_stv = mct_avect_indexra(o2x,'So_stv') index_o2x_So_rhoeff = mct_avect_indexra(o2x,'So_rhoeff') + index_o2x_So_tf2d = mct_avect_indexra(o2x,'So_tf2d',perrWith='quiet') index_o2x_So_algae1 = mct_avect_indexra(o2x,'So_algae1',perrWith='quiet') index_o2x_So_algae2 = mct_avect_indexra(o2x,'So_algae2',perrWith='quiet') diff --git a/components/mpas-ocean/driver/ocn_comp_mct.F b/components/mpas-ocean/driver/ocn_comp_mct.F index d1b140563bb7..31dc99904ebe 100644 --- a/components/mpas-ocean/driver/ocn_comp_mct.F +++ b/components/mpas-ocean/driver/ocn_comp_mct.F @@ -69,6 +69,7 @@ module ocn_comp_mct use ocn_config use ocn_submesoscale_eddies use ocn_eddy_parameterization_helpers + use ocn_scaled_dismf ! ! !PUBLIC MEMBER FUNCTIONS: implicit none @@ -108,6 +109,8 @@ module ocn_comp_mct integer :: nsend, nrecv + logical :: ocn_c2_glctf ! .true. => ocn to glc thermal forcing coupling on + character(len=StrKIND) :: runtype, coupleTimeStamp type(seq_infodata_type), pointer :: infodata @@ -214,6 +217,7 @@ subroutine ocn_init_mct( EClock, cdata_o, x2o_o, o2x_o, NLFilename )!{{{ character(len=StrKIND) :: iotype logical :: streamsExists integer :: mesh_iotype + logical :: ocn_c2_glcshelf logical, pointer :: tempLogicalConfig character(len=StrKIND), pointer :: tempCharConfig @@ -222,11 +226,15 @@ subroutine ocn_init_mct( EClock, cdata_o, x2o_o, o2x_o, NLFilename )!{{{ logical, pointer :: config_use_CFCTracers logical, pointer :: config_use_activeTracers_surface_restoring logical, pointer :: config_use_surface_salinity_monthly_restoring + logical, pointer :: config_scale_dismf_by_removed_ice_runoff character (len=StrKIND), pointer :: config_land_ice_flux_mode + character (len=StrKIND), pointer :: config_glc_thermal_forcing_coupling_mode ! ssh coupling interval initialization integer, pointer :: index_avgZonalSSHGradient, index_avgMeridionalSSHGradient real (kind=RKIND), dimension(:,:), pointer :: avgSSHGradient + real (kind=RKIND), pointer :: & + runningMeanRemovedIceRunoff ! the area integrated, running mean of removed ice runoff from the ocean #ifdef HAVE_MOAB character*100 outfile, wopts @@ -304,6 +312,9 @@ end subroutine xml_stream_get_attributes ! Determine coupling type call seq_infodata_GetData(infodata, cpl_seq_option=cpl_seq_option) + ! Determine if ocn to glc thermal forcing coupling is on + call seq_infodata_GetData(infodata, ocn_c2_glctf=ocn_c2_glctf) + !----------------------------------------------------------------------- ! ! initialize the model run @@ -815,6 +826,9 @@ end subroutine xml_stream_get_attributes call ocn_time_average_coupled_accumulate(statePool, forcingPool, 1) block_ptr => block_ptr % next end do + + ! initialize scaled data ice-shelf melt fluxes based on remove ice runoff + call ocn_init_scaled_dismf(domain) end if !----------------------------------------------------------------------- @@ -860,6 +874,9 @@ end subroutine xml_stream_get_attributes call ocn_time_average_coupled_accumulate(statePool, forcingPool, 1) block_ptr => block_ptr % next end do + + ! initialize scaled data ice-shelf melt fluxes based on remove ice runoff + call ocn_init_scaled_dismf(domain) end if call t_stopf ('mpaso_mct_init') @@ -869,14 +886,35 @@ end subroutine xml_stream_get_attributes trim(config_land_ice_flux_mode) == 'pressure_only' .or. & trim(config_land_ice_flux_mode) == 'data' .or. & trim(config_land_ice_flux_mode) == 'standalone' ) then - call seq_infodata_PutData( infodata, ocn_prognostic=.true., ocnrof_prognostic=.true., & - ocn_c2_glcshelf=.false.) + ocn_c2_glcshelf = .false. else if ( trim(config_land_ice_flux_mode) .eq. 'coupled' ) then - call seq_infodata_PutData( infodata, ocn_prognostic=.true., ocnrof_prognostic=.true., & - ocn_c2_glcshelf=.true.) + ocn_c2_glcshelf = .true. else call mpas_log_write('ERROR: unknown land_ice_flux_mode: ' // trim(config_land_ice_flux_mode), MPAS_LOG_CRIT) end if + call seq_infodata_PutData(infodata, ocn_prognostic=.true., ocnrof_prognostic=.true., & + ocn_c2_glcshelf=ocn_c2_glcshelf) + + call mpas_pool_get_config(domain % configs, 'config_scale_dismf_by_removed_ice_runoff', & + config_scale_dismf_by_removed_ice_runoff) + if (config_scale_dismf_by_removed_ice_runoff) then + ! independent of space so should be no need to loop over blocks + block_ptr => domain % blocklist + call mpas_pool_get_subpool(block_ptr % structs, 'forcing', forcingPool) + call MPAS_pool_get_array(forcingPool, "runningMeanRemovedIceRunoff", & + runningMeanRemovedIceRunoff) + call seq_infodata_PutData(infodata, rmean_rmv_ice_runoff=runningMeanRemovedIceRunoff) + end if + + call mpas_pool_get_config(domain % configs, 'config_glc_thermal_forcing_coupling_mode', config_glc_thermal_forcing_coupling_mode) + if ( trim(config_glc_thermal_forcing_coupling_mode) == 'off' ) then + call seq_infodata_PutData(infodata, ocn_c2_glctf=.false.) + else if ( trim(config_glc_thermal_forcing_coupling_mode) == '2d' ) then + call seq_infodata_PutData(infodata, ocn_c2_glctf=.true.) + else + call mpas_log_write('ERROR: unknown config_glc_thermal_forcing_coupling_mode: ' // & + trim(config_glc_thermal_forcing_coupling_mode), MPAS_LOG_CRIT) + end if !----------------------------------------------------------------------- ! @@ -884,13 +922,7 @@ end subroutine xml_stream_get_attributes ! !----------------------------------------------------------------------- - call ocn_import_mct(x2o_o, errorCode) - if (errorCode /= 0) then - call mpas_log_write('Error in ocn_import_mct', MPAS_LOG_CRIT) - endif - #ifdef HAVE_MOAB - #ifdef MOABCOMP ! loop over all fields in seq_flds_x2o_fields call mct_list_init(temp_list ,seq_flds_x2o_fields) @@ -906,6 +938,14 @@ end subroutine xml_stream_get_attributes enddo call mct_list_clean(temp_list) #endif +#endif + + call ocn_import_mct(x2o_o, errorCode) + if (errorCode /= 0) then + call mpas_log_write('Error in ocn_import_mct', MPAS_LOG_CRIT) + endif + +#ifdef HAVE_MOAB call ocn_import_moab(Eclock, errorCode) if (errorCode /= 0) then @@ -987,12 +1027,15 @@ subroutine ocn_run_mct( EClock, cdata_o, x2o_o, o2x_o)!{{{ logical, pointer :: config_use_CFCTracers logical, pointer :: config_use_activeTracers_surface_restoring logical, pointer :: config_use_surface_salinity_monthly_restoring + logical, pointer :: config_scale_dismf_by_removed_ice_runoff character (len=StrKIND), pointer :: config_restart_timestamp_name character (len=StrKIND), pointer :: config_sw_absorption_type ! Added for coupling interval initialization integer, pointer :: index_avgZonalSSHGradient, index_avgMeridionalSSHGradient real (kind=RKIND), dimension(:,:), pointer :: avgSSHGradient + real (kind=RKIND), pointer :: & + runningMeanRemovedIceRunoff ! the area integrated, running mean of removed ice runoff from the ocean #ifdef HAVE_MOAB #ifdef MOABCOMP @@ -1029,12 +1072,8 @@ subroutine ocn_run_mct( EClock, cdata_o, x2o_o, o2x_o)!{{{ call mpas_get_timeInterval(timeStep, dt=dt) call mpas_reset_clock_alarm(domain_ptr % clock, coupleAlarmID, ierr=ierr) - ! Import state from coupler - call ocn_import_mct(x2o_o, ierr) - ! Import state from moab coupler -#ifdef HAVE_MOAB - +#ifdef HAVE_MOAB #ifdef MOABCOMP ! loop over all fields in seq_flds_x2o_fields call mct_list_init(temp_list ,seq_flds_x2o_fields) @@ -1048,8 +1087,14 @@ subroutine ocn_run_mct( EClock, cdata_o, x2o_o, o2x_o)!{{{ call seq_comm_compare_mb_mct(modelStr, mpicom_moab, x2o_o, mct_field, MPOID, tagname, ent_type, difference) enddo call mct_list_clean(temp_list) +#endif #endif + ! Import state from coupler + call ocn_import_mct(x2o_o, ierr) + ! Import state from moab coupler +#ifdef HAVE_MOAB + call ocn_import_moab(Eclock, ierr) if (ierr /= 0) then call mpas_log_write('Error in ocn_import_moab', MPAS_LOG_CRIT) @@ -1193,6 +1238,9 @@ subroutine ocn_run_mct( EClock, cdata_o, x2o_o, o2x_o)!{{{ block_ptr => block_ptr % next end do + ! update scaled data ice-shelf melt fluxes based on remove ice runoff + call ocn_update_scaled_dismf(domain) + if (debugOn) call mpas_log_write(' Computing analysis members') call ocn_analysis_compute(domain_ptr, ierr) if (iam==0.and.debugOn) then @@ -1290,6 +1338,17 @@ subroutine ocn_run_mct( EClock, cdata_o, x2o_o, o2x_o)!{{{ #endif call check_clocks_sync(domain % clock, Eclock, ierr) + call mpas_pool_get_config(domain % configs, 'config_scale_dismf_by_removed_ice_runoff', & + config_scale_dismf_by_removed_ice_runoff) + if (config_scale_dismf_by_removed_ice_runoff) then + ! independent of space so should be no need to loop over blocks + block_ptr => domain % blocklist + call mpas_pool_get_subpool(block_ptr % structs, 'forcing', forcingPool) + call MPAS_pool_get_array(forcingPool, "runningMeanRemovedIceRunoff", & + runningMeanRemovedIceRunoff) + call seq_infodata_PutData(infodata, rmean_rmv_ice_runoff=runningMeanRemovedIceRunoff) + end if + ! Reset I/O logs call shr_file_setLogUnit (shrlogunit) call shr_file_setLogLevel(shrloglev) @@ -2138,7 +2197,7 @@ subroutine ocn_import_mct(x2o_o, errorCode)!{{{ call shr_sys_abort ('Error: incoming rofi_F is negative') end if if (config_remove_ais_ice_runoff) then - if (latCell(i) < -1.04719666667_RKIND) then ! 60S in radians + if (latCell(i) < -0.99483767345_RKIND) then ! 57S in radians removedIceRunoffFlux(i) = iceRunoffFlux(i) iceRunoffFlux(i) = 0.0_RKIND removedIceRunoffFluxThisProc = removedIceRunoffFluxThisProc + removedIceRunoffFlux(i) @@ -2689,7 +2748,8 @@ subroutine ocn_export_mct(o2x_o, errorCode) !{{{ avgRemovedRiverRunoffFlux, & avgRemovedIceRunoffFlux, & avgLandIceHeatFlux, & - avgRemovedIceRunoffHeatFlux + avgRemovedIceRunoffHeatFlux, & + avgThermalForcingAtCritDepth real (kind=RKIND), dimension(:,:), pointer :: avgTracersSurfaceValue, avgSurfaceVelocity, & avgSSHGradient, avgOceanSurfacePhytoC, & @@ -2708,6 +2768,7 @@ subroutine ocn_export_mct(o2x_o, errorCode) !{{{ config_use_MacroMoleculesTracers_sea_ice_coupling character (len=StrKIND), pointer :: config_land_ice_flux_mode + character (len=StrKIND), pointer :: config_glc_thermal_forcing_coupling_mode logical :: keepFrazil @@ -2718,6 +2779,8 @@ subroutine ocn_export_mct(o2x_o, errorCode) !{{{ call mpas_pool_get_config(domain % configs, 'config_land_ice_flux_mode', config_land_ice_flux_mode) call mpas_pool_get_config(domain % configs, 'config_remove_ais_river_runoff', config_remove_ais_river_runoff) call mpas_pool_get_config(domain % configs, 'config_remove_ais_ice_runoff', config_remove_ais_ice_runoff) + call mpas_pool_get_config(domain % configs, 'config_glc_thermal_forcing_coupling_mode', & + config_glc_thermal_forcing_coupling_mode) call mpas_pool_get_config(domain % configs, 'config_use_DMSTracers', config_use_DMSTracers) call mpas_pool_get_config(domain % configs, 'config_use_MacroMoleculesTracers', config_use_MacroMoleculesTracers) call mpas_pool_get_config(domain % configs, 'config_use_ecosysTracers_sea_ice_coupling', & @@ -2772,6 +2835,9 @@ subroutine ocn_export_mct(o2x_o, errorCode) !{{{ call mpas_pool_get_array(forcingPool, 'avgRemovedIceRunoffFlux', avgRemovedIceRunoffFlux) call mpas_pool_get_array(forcingPool, 'avgRemovedIceRunoffHeatFlux', avgRemovedIceRunoffHeatFlux) endif + if (trim(config_glc_thermal_forcing_coupling_mode) == '2d') then + call mpas_pool_get_array(forcingPool, 'avgThermalForcingAtCritDepth', avgThermalForcingAtCritDepth) + endif ! BGC fields if (config_use_ecosysTracers) then @@ -2933,6 +2999,10 @@ subroutine ocn_export_mct(o2x_o, errorCode) !{{{ o2x_o % rAttr(index_o2x_So_stv, n) = landIceTracerTransferVelocities(indexSaltTrans,i) o2x_o % rAttr(index_o2x_So_rhoeff, n) = 0.0_RKIND endif + if (trim(config_glc_thermal_forcing_coupling_mode) == '2d' .and. ocn_c2_glctf) then + o2x_o % rAttr(index_o2x_So_tf2d, n) = avgThermalForcingAtCritDepth(i) + endif + !Fyke: test !write(stderrUnit,*) 'n=',n @@ -3146,15 +3216,19 @@ end subroutine datetime!}}} #ifdef HAVE_MOAB -! import method from moab -! copied from ocn_import_mct, will replace x2o_o AV with x2o_om array read locally - subroutine ocn_import_moab( Eclock, errorCode)!{{{ + +!*********************************************************************** +!BOP +! !IROUTINE: ocn_import_moab +! !INTERFACE: + + subroutine ocn_import_moab(Eclock, errorCode)!{{{ ! !DESCRIPTION: !----------------------------------------------------------------------- -! This routine receives message from cpl7 driver +! This routine receives message from moab driver ! -! The following fields are always received from the coupler: +! The following fields are always received from the driver: ! ! o taux -- zonal wind stress (taux) (W/m2 ) ! o tauy -- meridonal wind stress (tauy) (W/m2 ) @@ -3171,11 +3245,22 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ ! o ifrac -- ice fraction (%) ! o rofl -- river runoff flux (kg/m2/s) ! o rofi -- ice runoff flux (kg/m2/s) +! o rofDIN -- DIN runoff flux (kg/m2/s) +! o rofDIP -- DIP runoff flux (kg/m2/s) +! o rofDON -- DON runoff flux (kg/m2/s) +! o rofDOP -- DOP runoff flux (kg/m2/s) +! o rofDOC -- DOC runoff flux (kg/m2/s) +! o rofPP -- PP runoff flux (kg/m2/s) +! o rofDSi -- DSi runoff flux (kg/m2/s) +! o rofPOC -- POC runoff flux (kg/m2/s) +! o rofPN -- PN runoff flux (kg/m2/s) +! o rofDIC -- DIC runoff flux (kg/m2/s) +! o rofFe -- Fe runoff flux (kg/m2/s) ! ! The following fields are sometimes received from the coupler, ! depending on model options: ! -! o pbot -- bottom atm pressure (Pa) +! o pslv -- atmospheric pressure at sea level (Pa) ! o duu10n -- 10m wind speed squared (m^2/s^2) ! o co2prog-- bottom atm level prognostic co2 ! o co2diag-- bottom atm level diagnostic co2 @@ -3184,28 +3269,15 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ ! ! !REVISION HISTORY: ! same as module - + use iMOAB, only : iMOAB_GetDoubleTagStorage, iMOAB_WriteMesh ! !INPUT/OUTPUT PARAMETERS: + type(ESMF_Clock), intent(inout) :: EClock ! type(mct_aVect) , intent(inout) :: x2o_o ! instead, we will get x2o_om from MPOID ! !OUTPUT PARAMETERS: - use iMOAB, only : iMOAB_GetDoubleTagStorage, iMOAB_WriteMesh - !EOP - !BOC - !----------------------------------------------------------------------- - ! - ! local variables - !----------------------------------------------------------------------- - ! - ! local variables - ! - !----------------------------------------------------------------------- - integer :: ent_type, ierr - character(CXX) :: tagname - type(ESMF_Clock), intent(inout) :: EClock integer, intent(out) :: & errorCode ! returned error code @@ -3221,6 +3293,9 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ label, & message + integer :: ent_type, ierr + character(CXX) :: tagname + integer :: & i,n @@ -3232,6 +3307,7 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ config_use_DMSTracers_sea_ice_coupling, & config_use_MacroMoleculesTracers, & config_use_MacroMoleculesTracers_sea_ice_coupling, & + config_use_CFCTracers, & config_remove_ais_river_runoff, & config_remove_ais_ice_runoff, & config_cvmix_kpp_use_theory_wave @@ -3250,7 +3326,8 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ ecosysAuxiliary, & ecosysSeaIceCoupling, & DMSSeaIceCoupling, & - MacroMoleculesSeaIceCoupling + MacroMoleculesSeaIceCoupling, & + CFCAuxiliary integer, pointer :: nCellsSolve @@ -3278,10 +3355,22 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ iceFluxFeParticulateField, & iceFluxFeDissolvedField, & iceFluxDustField, & + riverFluxNO3Field, & + riverFluxPO4Field, & + riverFluxSiO3Field, & + riverFluxDOCField, & + riverFluxDONField, & + riverFluxDOPField, & + riverFluxDICField, & + riverFluxALKField, & + riverFluxFeField, & landIceFreshwaterFluxField, & landIceHeatFluxField, & landIceFractionField, & - windSpeed10mField + windSpeed10mField, & + significantWaveHeightField, & + peakWaveFrequencyField, & + peakWaveDirectionField !landIcePressureField type (field2DReal), pointer :: iceFluxPhytoCField, & @@ -3289,6 +3378,9 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ type (field2DReal), pointer :: landIceInterfaceTracersField + type (field2DReal), pointer :: stokesDriftZonalWavenumberField, & + stokesDriftMeridionalWavenumberField + real (kind=RKIND), dimension(:), pointer :: windStressZonal, windStressMeridional, & latentHeatFlux, sensibleHeatFlux, & longWaveHeatFluxUp, & @@ -3302,6 +3394,7 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ atmosphericPressure, iceFraction, & seaIcePressure, windSpeedSquared10m, & atmosphericCO2, atmosphericCO2_ALT_CO2, & + windSpeedSquared10mCFC, & iceFluxDIC, & iceFluxDON, & iceFluxNO3, & @@ -3313,26 +3406,44 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ iceFluxFeParticulate, & iceFluxFeDissolved, & iceFluxDust, & + riverFluxNO3, & + riverFluxPO4, & + riverFluxSiO3, & + riverFluxDOC, & + riverFluxDON, & + riverFluxDOP, & + riverFluxDIC, & + riverFluxALK, & + riverFluxFe, & landIceFreshwaterFlux, & landIceHeatFlux, & landIceFraction, & - windSpeed10m + areaCell, & + windSpeed10m, & + significantWaveHeight, & + peakWaveFrequency, & + peakWaveDirection !landIcePressure real (kind=RKIND), dimension(:), pointer :: latCell real (kind=RKIND), dimension(:,:), pointer :: iceFluxPhytoC, & - iceFluxDOC + iceFluxDOC, & + stokesDriftZonalWavenumber, & + stokesDriftMeridionalWavenumber real (kind=RKIND) :: removedRiverRunoffFluxThisProc, removedIceRunoffFluxThisProc real (kind=RKIND) :: removedRiverRunoffFluxReduced, removedIceRunoffFluxReduced real (kind=RKIND), dimension(:,:), pointer :: landIceInterfaceTracers + real (kind=RKIND) :: riverFactor + !----------------------------------------------------------------------- ! ! zero out padded cells ! +!----------------------------------------------------------------------- !----------------------------------------------------------------------- integer :: cur_ocn_stepno #ifdef MOABDEBUG @@ -3349,11 +3460,10 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ write(ocnLogUnit,*) 'Fail to write ocean state ' endif #endif + errorCode = 0 ! get moab tags from MPOID - - ent_type = 1 ! cells ! get all tags in one method tagname = trim(seq_flds_x2o_fields)//C_NULL_CHAR @@ -3361,7 +3471,6 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ if ( ierr /= 0 ) then write(ocnLogUnit,*) 'Fail to get MOAB fields ' endif - !----------------------------------------------------------------------- ! ! unpack and distribute wind stress, then convert to correct units @@ -3380,6 +3489,7 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ config_use_DMSTracers_sea_ice_coupling) call mpas_pool_get_config(domain % configs, 'config_use_MacroMoleculesTracers_sea_ice_coupling', & config_use_MacroMoleculesTracers_sea_ice_coupling) + call mpas_pool_get_config(domain % configs, 'config_use_CFCTracers', config_use_CFCTracers) call mpas_pool_get_config(domain % configs, 'config_remove_ais_river_runoff', config_remove_ais_river_runoff) call mpas_pool_get_config(domain % configs, 'config_remove_ais_ice_runoff', config_remove_ais_ice_runoff) call mpas_pool_get_config(domain % configs, 'config_cvmix_kpp_use_theory_wave', config_cvmix_kpp_use_theory_wave) @@ -3418,6 +3528,11 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ call mpas_pool_get_field(forcingPool, 'iceRunoffFlux', iceRunoffFluxField) call mpas_pool_get_field(forcingPool, 'removedRiverRunoffFlux', removedRiverRunoffFluxField) call mpas_pool_get_field(forcingPool, 'removedIceRunoffFlux', removedIceRunoffFluxField) + call mpas_pool_get_field(forcingPool, 'stokesDriftZonalWavenumber', stokesDriftZonalWavenumberField) + call mpas_pool_get_field(forcingPool, 'stokesDriftMeridionalWavenumber', stokesDriftMeridionalWavenumberField) + call mpas_pool_get_field(forcingPool, 'significantWaveHeight', significantWaveHeightField) + call mpas_pool_get_field(forcingPool, 'peakWaveFrequency', peakWaveFrequencyField) + call mpas_pool_get_field(forcingPool, 'peakWaveDirection', peakWaveDirectionField) call mpas_pool_get_field(forcingPool, 'landIceFreshwaterFlux', landIceFreshwaterFluxField) call mpas_pool_get_field(forcingPool, 'landIceHeatFlux', landIceHeatFluxField) @@ -3459,9 +3574,15 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ landIceInterfaceTracers => landIceInterfaceTracersField % array landIceFraction => landIceFractionField % array windSpeed10m => windSpeed10mField % array + stokesDriftZonalWavenumber => stokesDriftZonalWavenumberField % array + stokesDriftMeridionalWavenumber => stokesDriftMeridionalWavenumberField % array + significantWaveHeight => significantWaveHeightField % array + peakWaveFrequency => peakWaveFrequencyField % array + peakWaveDirection => peakWaveDirectionField % array !landIcePressure => landIcePressureField % array call mpas_pool_get_array(meshPool, 'latCell', latCell) + call mpas_pool_get_array(meshPool, 'areaCell', areaCell) ! BGC fields if (config_use_ecosysTracers) then @@ -3474,6 +3595,27 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ call mpas_pool_get_field(ecosysAuxiliary, 'atmosphericCO2_ALT_CO2', atmosphericCO2_ALT_CO2Field) atmosphericCO2_ALT_CO2 => atmosphericCO2_ALT_CO2Field % array + if (config_use_ecosysTracers_river_inputs_from_coupler) then + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxNO3' , riverFluxNO3Field) + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxPO4' , riverFluxPO4Field) + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxDON' , riverFluxDONField) + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxDOP' , riverFluxDOPField) + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxSiO3', riverFluxSiO3Field) + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxDOC' , riverFluxDOCField) + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxDIC' , riverFluxDICField) + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxALK' , riverFluxALKField) + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxFe' , riverFluxFeField) + riverFluxNO3 => riverFluxNO3Field % array + riverFluxPO4 => riverFluxPO4Field % array + riverFluxDON => riverFluxDONField % array + riverFluxDOP => riverFluxDOPField % array + riverFluxSiO3 => riverFluxSiO3Field % array + riverFluxDOC => riverFluxDOCField % array + riverFluxDIC => riverFluxDICField % array + riverFluxALK => riverFluxALKField % array + riverFluxFe => riverFluxFeField % array + endif + call mpas_pool_get_config(domain % configs, 'config_ecosys_atm_co2_option', & config_ecosys_atm_co2_option) call mpas_pool_get_config(domain % configs, 'config_ecosys_atm_alt_co2_option', & @@ -3519,6 +3661,13 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ iceFluxDMSP => iceFluxDMSPField % array endif + ! CFC fields + if (config_use_CFCTracers) then + call mpas_pool_get_subpool(forcingPool, 'CFCAuxiliary', CFCAuxiliary) + call mpas_pool_get_field(CFCAuxiliary, 'windSpeedSquared10mCFC', windSpeedSquared10mField) + windSpeedSquared10mCFC => windSpeedSquared10mField % array + endif + if (config_remove_ais_river_runoff) then ! Initialize this field removedRiverRunoffFlux(:) = 0.0_RKIND @@ -3533,8 +3682,7 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ ! Initialize this field windSpeed10m(:) = 0.0_RKIND endif - -! replace 'x2o_o % rAttr(' to 'x2o_om(n, ' and ', n)' with ')' +! ! replace 'x2o_o % rAttr(' to 'x2o_om(n, ' and ', n)' with ')' do i = 1, nCellsSolve n = n + 1 if ( windStressZonalField % isActive ) then @@ -3593,7 +3741,7 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ call shr_sys_abort ('Error: incoming rofi_F is negative') end if if (config_remove_ais_ice_runoff) then - if (latCell(i) < -1.04719666667_RKIND) then ! 60S in radians + if (latCell(i) < -0.99483767345_RKIND) then ! 57S in radians removedIceRunoffFlux(i) = iceRunoffFlux(i) iceRunoffFlux(i) = 0.0_RKIND removedIceRunoffFluxThisProc = removedIceRunoffFluxThisProc + removedIceRunoffFlux(i) @@ -3618,6 +3766,32 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ iceFraction(i) = x2o_om(n, index_x2o_Si_ifrac) end if + if ( stokesDriftZonalWavenumberField % isActive ) then + stokesDriftZonalWavenumber(1,i) = x2o_om(n, index_x2o_Sw_ustokes_wavenumber_1) + stokesDriftZonalWavenumber(2,i) = x2o_om(n, index_x2o_Sw_ustokes_wavenumber_2) + stokesDriftZonalWavenumber(3,i) = x2o_om(n, index_x2o_Sw_ustokes_wavenumber_3) + stokesDriftZonalWavenumber(4,i) = x2o_om(n, index_x2o_Sw_ustokes_wavenumber_4) + stokesDriftZonalWavenumber(5,i) = x2o_om(n, index_x2o_Sw_ustokes_wavenumber_5) + stokesDriftZonalWavenumber(6,i) = x2o_om(n, index_x2o_Sw_ustokes_wavenumber_6) + end if + if ( stokesDriftMeridionalWavenumberField % isActive ) then + stokesDriftMeridionalWavenumber(1,i) = x2o_om(n, index_x2o_Sw_vstokes_wavenumber_1) + stokesDriftMeridionalWavenumber(2,i) = x2o_om(n, index_x2o_Sw_vstokes_wavenumber_2) + stokesDriftMeridionalWavenumber(3,i) = x2o_om(n, index_x2o_Sw_vstokes_wavenumber_3) + stokesDriftMeridionalWavenumber(4,i) = x2o_om(n, index_x2o_Sw_vstokes_wavenumber_4) + stokesDriftMeridionalWavenumber(5,i) = x2o_om(n, index_x2o_Sw_vstokes_wavenumber_5) + stokesDriftMeridionalWavenumber(6,i) = x2o_om(n, index_x2o_Sw_vstokes_wavenumber_6) + end if + if ( significantWaveHeightField % isActive ) then + significantWaveHeight(i) = x2o_om(n, index_x2o_Sw_Hs) + end if + if ( peakWaveFrequencyField % isActive ) then + peakWaveFrequency(i) = x2o_om(n, index_x2o_Sw_Fp) + end if + if ( peakWaveDirectionField % isActive ) then + peakWaveDirection(i) = x2o_om(n, index_x2o_Sw_Dp) + end if + if (config_cvmix_kpp_use_theory_wave) then if ( windSpeed10mField% isActive ) then windSpeed10m(i) = sqrt( x2o_om(n, index_x2o_So_duu10n)) @@ -3675,12 +3849,36 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ else if ( config_ecosys_atm_alt_co2_option == 'bdrc') then atmosphericCO2_ALT_CO2(i) = config_ecosys_atm_co2_constant_value else if ( config_ecosys_atm_alt_co2_option == 'bdrd') then - atmosphericCO2_ALT_CO2(i) = x2o_om(n, index_x2o_Sa_co2diag) + atmosphericCO2_ALT_CO2(i) = x2o_om(n, index_x2o_Sa_co2diag) else atmosphericCO2_ALT_CO2(i) = config_ecosys_atm_co2_constant_value end if end if + if (config_use_ecosysTracers_river_inputs_from_coupler) then + riverFluxNO3(i) = x2o_om(n, index_x2o_Foxx_rofDIN) + riverFluxPO4(i) = x2o_om(n, index_x2o_Foxx_rofDIP) + riverFluxDON(i) = x2o_om(n, index_x2o_Foxx_rofDON) + riverFluxDOP(i) = x2o_om(n, index_x2o_Foxx_rofDOP) + riverFluxSiO3(i) = x2o_om(n, index_x2o_Foxx_rofDSi) + riverFluxDOC(i) = x2o_om(n, index_x2o_Foxx_rofDOC) + riverFluxDIC(i) = x2o_om(n, index_x2o_Foxx_rofDIC) + riverFluxFe(i) = x2o_om(n, index_x2o_Foxx_rofFe ) + +! convert from kgNutrient/(m2-s) to mmol/m3 m/s + riverFactor = 1.e6_RKIND + riverFluxNO3(i) = riverFluxNO3(i)*riverFactor/14.007_RKIND + riverFluxPO4(i) = riverFluxPO4(i)*riverFactor/30.974_RKIND + riverFluxDON(i) = riverFluxDON(i)*riverFactor/14.007_RKIND + riverFluxDOP(i) = riverFluxDOP(i)*riverFactor/30.974_RKIND + riverFluxSiO3(i) = riverFluxSiO3(i)*riverFactor/28.085_RKIND + riverFluxDOC(i) = riverFluxDOC(i)*riverFactor/12.001_RKIND + riverFluxDIC(i) = riverFluxDIC(i)*riverFactor/12.001_RKIND + riverFluxFe(i) = riverFluxFe(i)*riverFactor/55.845_RKIND + + riverFluxALK(i) = riverFluxDIC(i) + endif + if (config_use_ecosysTracers_sea_ice_coupling) then if ( iceFluxPhytoCField % isActive ) then iceFluxPhytoC(1,i) = x2o_om(n, index_x2o_Fioi_algae1) @@ -3714,6 +3912,7 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ if ( iceFluxDOCField % isActive ) then iceFluxDOC(1,i) = x2o_om(n, index_x2o_Fioi_doc1) iceFluxDOC(2,i) = x2o_om(n, index_x2o_Fioi_doc2) + iceFluxDOC(3,i) = x2o_om(n, index_x2o_Fioi_doc3) endif if ( iceFluxDONField % isActive ) then iceFluxDON(i) = x2o_om(n, index_x2o_Fioi_don1) @@ -3730,6 +3929,13 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ endif endif + ! CFC fields + if (config_use_CFCTracers) then + if ( windSpeedSquared10mField % isActive ) then + windSpeedSquared10mCFC(i) = x2o_om(n, index_x2o_So_duu10n) + end if + end if + end do block_ptr => block_ptr % next @@ -3757,6 +3963,11 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ call mpas_pool_get_field(forcingPool, 'atmosphericPressure', atmosphericPressureField) call mpas_pool_get_field(forcingPool, 'seaIcePressure', seaIcePressureField) call mpas_pool_get_field(forcingPool, 'iceFraction', iceFractionField) + call mpas_pool_get_field(forcingPool, 'stokesDriftZonalWavenumber', stokesDriftZonalWavenumberField) + call mpas_pool_get_field(forcingPool, 'stokesDriftMeridionalWavenumber', stokesDriftMeridionalWavenumberField) + call mpas_pool_get_field(forcingPool, 'significantWaveHeight', significantWaveHeightField) + call mpas_pool_get_field(forcingPool, 'peakWaveFrequency', peakWaveFrequencyField) + call mpas_pool_get_field(forcingPool, 'peakWaveDirection', peakWaveDirectionField) call mpas_pool_get_field(forcingPool, 'landIceFreshwaterFlux', landIceFreshwaterFluxField) call mpas_pool_get_field(forcingPool, 'landIceHeatFlux', landIceHeatFluxField) @@ -3777,6 +3988,18 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ call mpas_pool_get_field(ecosysAuxiliary, 'atmosphericCO2', atmosphericCO2Field) call mpas_pool_get_field(ecosysAuxiliary, 'atmosphericCO2_ALT_CO2', atmosphericCO2_ALT_CO2Field) + if (config_use_ecosysTracers_river_inputs_from_coupler) then + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxNO3' , riverFluxNO3Field) + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxPO4' , riverFluxPO4Field) + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxDON' , riverFluxDONField) + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxDOP' , riverFluxDOPField) + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxSiO3', riverFluxSiO3Field) + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxDOC' , riverFluxDOCField) + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxDIC' , riverFluxDICField) + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxALK' , riverFluxALKField) + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxFe' , riverFluxFeField) + endif + if (config_use_ecosysTracers_sea_ice_coupling) then call mpas_pool_get_subpool(forcingPool, 'ecosysSeaIceCoupling', ecosysSeaIceCoupling) @@ -3800,6 +4023,12 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ call mpas_pool_get_field(DMSSeaIceCoupling, 'iceFluxDMSP', iceFluxDMSPField) endif + ! CFC fields + if (config_use_CFCTracers) then + call mpas_pool_get_subpool(forcingPool, 'CFCAuxiliary', CFCAuxiliary) + call mpas_pool_get_field(CFCAuxiliary, 'windSpeedSquared10mCFC', windSpeedSquared10mField) + endif + if ( windStressMeridionalField % isActive ) then call mpas_dmpar_exch_halo_field(windStressMeridionalField) end if @@ -3860,6 +4089,21 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ if ( iceFractionField % isActive ) then call mpas_dmpar_exch_halo_field(iceFractionField) end if + if ( stokesDriftZonalWavenumberField % isActive ) then + call mpas_dmpar_exch_halo_field(stokesDriftZonalWavenumberField) + end if + if ( stokesDriftMeridionalWavenumberField % isActive ) then + call mpas_dmpar_exch_halo_field(stokesDriftMeridionalWavenumberField) + end if + if ( significantWaveHeightField % isActive ) then + call mpas_dmpar_exch_halo_field(significantWaveHeightField) + end if + if ( peakWaveFrequencyField % isActive ) then + call mpas_dmpar_exch_halo_field(peakWaveFrequencyField) + end if + if ( peakWaveDirectionField % isActive ) then + call mpas_dmpar_exch_halo_field(peakWaveDirectionField) + end if if ( landIceFreshwaterFluxField % isActive ) then call mpas_dmpar_exch_halo_field(landIceFreshwaterFluxField) @@ -3894,32 +4138,62 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ call mpas_dmpar_exch_halo_field(atmosphericCO2_ALT_CO2Field) end if - if (config_use_ecosysTracers_sea_ice_coupling) then - if ( iceFluxPhytoCField % isActive ) then - call mpas_dmpar_exch_halo_field(iceFluxPhytoCField) - endif - if ( iceFluxDICField % isActive ) then - call mpas_dmpar_exch_halo_field(iceFluxDICField) - endif - if ( iceFluxNO3Field % isActive ) then - call mpas_dmpar_exch_halo_field(iceFluxNO3Field) - endif - if ( iceFluxSiO3Field % isActive ) then - call mpas_dmpar_exch_halo_field(iceFluxSiO3Field) - endif - if ( iceFluxNH4Field % isActive ) then - call mpas_dmpar_exch_halo_field(iceFluxNH4Field) - endif - if ( iceFluxDOCrField % isActive ) then - call mpas_dmpar_exch_halo_field(iceFluxDOCrField) - endif - if ( iceFluxFeParticulateField % isActive ) then - call mpas_dmpar_exch_halo_field(iceFluxFeParticulateField) - endif - if ( iceFluxFeDissolvedField % isActive ) then - call mpas_dmpar_exch_halo_field(iceFluxFeDissolvedField) - endif - if ( iceFluxDustField % isActive ) then + if (config_use_ecosysTracers_river_inputs_from_coupler) then + if ( riverFluxNO3Field % isActive ) then + call mpas_dmpar_exch_halo_field(riverFluxNO3Field) + end if + if ( riverFluxPO4Field % isActive ) then + call mpas_dmpar_exch_halo_field(riverFluxPO4Field) + end if + if ( riverFluxDONField % isActive ) then + call mpas_dmpar_exch_halo_field(riverFluxDONField) + end if + if ( riverFluxDOPField % isActive ) then + call mpas_dmpar_exch_halo_field(riverFluxDOPField) + end if + if ( riverFluxSiO3Field % isActive ) then + call mpas_dmpar_exch_halo_field(riverFluxSiO3Field) + end if + if ( riverFluxDOCField % isActive ) then + call mpas_dmpar_exch_halo_field(riverFluxDOCField) + end if + if ( riverFluxDICField % isActive ) then + call mpas_dmpar_exch_halo_field(riverFluxDICField) + end if + if ( riverFluxALKField % isActive ) then + call mpas_dmpar_exch_halo_field(riverFluxALKField) + end if + if ( riverFluxFeField % isActive ) then + call mpas_dmpar_exch_halo_field(riverFluxFeField) + end if + endif + + if (config_use_ecosysTracers_sea_ice_coupling) then + if ( iceFluxPhytoCField % isActive ) then + call mpas_dmpar_exch_halo_field(iceFluxPhytoCField) + endif + if ( iceFluxDICField % isActive ) then + call mpas_dmpar_exch_halo_field(iceFluxDICField) + endif + if ( iceFluxNO3Field % isActive ) then + call mpas_dmpar_exch_halo_field(iceFluxNO3Field) + endif + if ( iceFluxSiO3Field % isActive ) then + call mpas_dmpar_exch_halo_field(iceFluxSiO3Field) + endif + if ( iceFluxNH4Field % isActive ) then + call mpas_dmpar_exch_halo_field(iceFluxNH4Field) + endif + if ( iceFluxDOCrField % isActive ) then + call mpas_dmpar_exch_halo_field(iceFluxDOCrField) + endif + if ( iceFluxFeParticulateField % isActive ) then + call mpas_dmpar_exch_halo_field(iceFluxFeParticulateField) + endif + if ( iceFluxFeDissolvedField % isActive ) then + call mpas_dmpar_exch_halo_field(iceFluxFeDissolvedField) + endif + if ( iceFluxDustField % isActive ) then call mpas_dmpar_exch_halo_field(iceFluxDustField) endif if ( iceFluxDOCField % isActive ) then @@ -3939,302 +4213,348 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ endif endif + ! CFC fields + if (config_use_CFCTracers .and. .not. config_use_ecosysTracers) then + if ( windSpeedSquared10mField % isActive ) then + call mpas_dmpar_exch_halo_field(windSpeedSquared10mField) + end if + endif + !----------------------------------------------------------------------- !EOC end subroutine ocn_import_moab!}}} +!*********************************************************************** +!BOP +! !IROUTINE: ocn_export_moab +! !INTERFACE: - subroutine ocn_export_moab(EClock) !{{{ - - ! !DESCRIPTION: - ! This routine calls the routines necessary to send mpas ocean fields to MOAB coupler - ! - use iMOAB, only : iMOAB_SetDoubleTagStorage, iMOAB_WriteMesh - !EOP - !BOC - type(ESMF_Clock) , intent(inout) :: EClock ! Input synchronization clock from driver - ! - ! local variables - ! - !----------------------------------------------------------------------- - integer :: ent_type, ierr, cur_ocn_stepno - character(len=100) :: outfile, wopts, localmeshfile, lnum - character(CXX) :: tagname - - integer :: i, n - integer, pointer :: nCellsSolve, index_temperatureSurfaceValue, index_salinitySurfaceValue, & - index_avgZonalSurfaceVelocity, index_avgMeridionalSurfaceVelocity, & - index_avgZonalSSHGradient, index_avgMeridionalSSHGradient + subroutine ocn_export_moab(EClock) !{{{ - type (block_type), pointer :: block_ptr +! !DESCRIPTION: +! This routine calls the routines necessary to send MPASO fields to +! the MOAB driver +! +! !REVISION HISTORY: +! same as module + use iMOAB, only : iMOAB_SetDoubleTagStorage, iMOAB_WriteMesh +! !INPUT/OUTPUT PARAMETERS: - type (mpas_pool_type), pointer :: meshPool, & - forcingPool, & - statePool, & - tracersPool, & - ecosysAuxiliary, & - ecosysSeaIceCoupling, & - DMSSeaIceCoupling, & - MacroMoleculesSeaIceCoupling - - integer, dimension(:), pointer :: landIceMask - - real (kind=RKIND), dimension(:), pointer :: seaIceEnergy, accumulatedFrazilIceMass, frazilSurfacePressure, & - avgTotalFreshWaterTemperatureFlux, & - avgCO2_gas_flux, DMSFlux, surfaceUpwardCO2Flux, & - avgOceanSurfaceDIC, & - avgOceanSurfaceDON, & - avgOceanSurfaceNO3, & - avgOceanSurfaceSiO3, & - avgOceanSurfaceNH4, & - avgOceanSurfaceDMS, & - avgOceanSurfaceDMSP, & - avgOceanSurfaceDOCr, & - avgOceanSurfaceDOCSemiLabile, & - avgOceanSurfaceFeParticulate, & - avgOceanSurfaceFeDissolved, & - ssh, & - avgLandIceFreshwaterFlux, & - avgRemovedRiverRunoffFlux, & - avgRemovedIceRunoffFlux, & - avgLandIceHeatFlux, & - avgRemovedIceRunoffHeatFlux - - real (kind=RKIND), dimension(:,:), pointer :: avgTracersSurfaceValue, avgSurfaceVelocity, & - avgSSHGradient, avgOceanSurfacePhytoC, & - avgOceanSurfaceDOC, layerThickness - - real (kind=RKIND) :: surfaceFreezingTemp - - logical, pointer :: frazilIceActive, & - config_remove_ais_river_runoff, & - config_remove_ais_ice_runoff, & - config_use_ecosysTracers, & - config_use_DMSTracers, & - config_use_MacroMoleculesTracers, & - config_use_ecosysTracers_sea_ice_coupling, & - config_use_DMSTracers_sea_ice_coupling, & - config_use_MacroMoleculesTracers_sea_ice_coupling - - character (len=StrKIND), pointer :: config_land_ice_flux_mode - - logical :: keepFrazil - - - ! get configure options - call mpas_pool_get_package(domain % packages, 'frazilIceActive', frazilIceActive) - call mpas_pool_get_config(domain % configs, 'config_use_ecosysTracers', config_use_ecosysTracers) - call mpas_pool_get_config(domain % configs, 'config_land_ice_flux_mode', config_land_ice_flux_mode) - call mpas_pool_get_config(domain % configs, 'config_remove_ais_river_runoff', config_remove_ais_river_runoff) - call mpas_pool_get_config(domain % configs, 'config_remove_ais_ice_runoff', config_remove_ais_ice_runoff) - call mpas_pool_get_config(domain % configs, 'config_use_DMSTracers', config_use_DMSTracers) - call mpas_pool_get_config(domain % configs, 'config_use_MacroMoleculesTracers', config_use_MacroMoleculesTracers) - call mpas_pool_get_config(domain % configs, 'config_use_ecosysTracers_sea_ice_coupling', & - config_use_ecosysTracers_sea_ice_coupling) - call mpas_pool_get_config(domain % configs, 'config_use_DMSTracers_sea_ice_coupling', & - config_use_DMSTracers_sea_ice_coupling) - call mpas_pool_get_config(domain % configs, 'config_use_MacroMoleculesTracers_sea_ice_coupling', & - config_use_MacroMoleculesTracers_sea_ice_coupling) - - n = 0 - block_ptr => domain % blocklist - do while(associated(block_ptr)) - call mpas_pool_get_subpool(block_ptr % structs, 'mesh', meshPool) - call mpas_pool_get_subpool(block_ptr % structs, 'forcing', forcingPool) - call mpas_pool_get_subpool(block_ptr % structs, 'state', statePool) + type(ESMF_Clock) , intent(inout) :: EClock ! Input synchronization clock from driver - call mpas_pool_get_subpool(statePool, 'tracers', tracersPool) +! !OUTPUT PARAMETERS: - call mpas_pool_get_dimension(meshPool, 'nCellsSolve', nCellsSolve) +!EOP +!BOC +!----------------------------------------------------------------------- +! +! local variables +! +!----------------------------------------------------------------------- + integer :: ent_type, ierr, cur_ocn_stepno + character(len=100) :: outfile, wopts, localmeshfile, lnum + character(CXX) :: tagname - call mpas_pool_get_dimension(forcingPool, 'index_avgTemperatureSurfaceValue', index_temperatureSurfaceValue) - call mpas_pool_get_dimension(forcingPool, 'index_avgSalinitySurfaceValue', index_salinitySurfaceValue) - call mpas_pool_get_dimension(forcingPool, 'index_avgSurfaceVelocityZonal', index_avgZonalSurfaceVelocity) - call mpas_pool_get_dimension(forcingPool, 'index_avgSurfaceVelocityMeridional', index_avgMeridionalSurfaceVelocity) - call mpas_pool_get_dimension(forcingPool, 'index_avgSSHGradientZonal', index_avgZonalSSHGradient) - call mpas_pool_get_dimension(forcingPool, 'index_avgSSHGradientMeridional', index_avgMeridionalSSHGradient) + integer :: i, n + integer, pointer :: nCellsSolve, index_temperatureSurfaceValue, index_salinitySurfaceValue, & + index_avgZonalSurfaceVelocity, index_avgMeridionalSurfaceVelocity, & + index_avgZonalSSHGradient, index_avgMeridionalSSHGradient + type (block_type), pointer :: block_ptr - call mpas_pool_get_array(statePool, 'ssh', ssh, 1) - call mpas_pool_get_array(statePool, 'layerThickness', layerThickness, 1) + type (mpas_pool_type), pointer :: meshPool, & + forcingPool, & + statePool, & + tracersPool, & + ecosysAuxiliary, & + ecosysSeaIceCoupling, & + DMSSeaIceCoupling, & + MacroMoleculesSeaIceCoupling - call mpas_pool_get_array(forcingPool, 'landIceMask', landIceMask) - call mpas_pool_get_array(forcingPool, 'avgTracersSurfaceValue', avgTracersSurfaceValue) - call mpas_pool_get_array(forcingPool, 'avgSurfaceVelocity', avgSurfaceVelocity) - call mpas_pool_get_array(forcingPool, 'avgSSHGradient', avgSSHGradient) - call mpas_pool_get_array(forcingPool, 'avgTotalFreshWaterTemperatureFlux', avgTotalFreshWaterTemperatureFlux) - if ( frazilIceActive ) then - call mpas_pool_get_array(forcingPool, 'seaIceEnergy', seaIceEnergy) - call mpas_pool_get_array(forcingPool, 'frazilSurfacePressure', frazilSurfacePressure) - call mpas_pool_get_array(statePool, 'accumulatedFrazilIceMass', accumulatedFrazilIceMass, 1) - end if + integer, dimension(:), pointer :: landIceMask - if (trim(config_land_ice_flux_mode) == 'standalone' .or. trim(config_land_ice_flux_mode) == 'data') then - call mpas_pool_get_array(forcingPool, 'avgLandIceFreshwaterFlux', avgLandIceFreshwaterFlux) - call mpas_pool_get_array(forcingPool, 'avgLandIceHeatFlux', avgLandIceHeatFlux) - endif - if (config_remove_ais_river_runoff) then - call mpas_pool_get_array(forcingPool, 'avgRemovedRiverRunoffFlux', avgRemovedRiverRunoffFlux) - endif - if (config_remove_ais_ice_runoff) then - call mpas_pool_get_array(forcingPool, 'avgRemovedIceRunoffFlux', avgRemovedIceRunoffFlux) - call mpas_pool_get_array(forcingPool, 'avgRemovedIceRunoffHeatFlux', avgRemovedIceRunoffHeatFlux) - endif + real (kind=RKIND), dimension(:), pointer :: seaIceEnergy, accumulatedFrazilIceMass, frazilSurfacePressure, & + avgTotalFreshWaterTemperatureFlux, & + avgCO2_gas_flux, DMSFlux, surfaceUpwardCO2Flux, & + avgOceanSurfaceDIC, & + avgOceanSurfaceDON, & + avgOceanSurfaceNO3, & + avgOceanSurfaceSiO3, & + avgOceanSurfaceNH4, & + avgOceanSurfaceDMS, & + avgOceanSurfaceDMSP, & + avgOceanSurfaceDOCr, & + avgOceanSurfaceDOCSemiLabile, & + avgOceanSurfaceFeParticulate, & + avgOceanSurfaceFeDissolved, & + ssh, & + avgLandIceFreshwaterFlux, & + avgRemovedRiverRunoffFlux, & + avgRemovedIceRunoffFlux, & + avgLandIceHeatFlux, & + avgRemovedIceRunoffHeatFlux - ! BGC fields - if (config_use_ecosysTracers) then + real (kind=RKIND), dimension(:,:), pointer :: avgTracersSurfaceValue, avgSurfaceVelocity, & + avgSSHGradient, avgOceanSurfacePhytoC, & + avgOceanSurfaceDOC, layerThickness - call mpas_pool_get_subpool(forcingPool, 'ecosysAuxiliary', ecosysAuxiliary) - call mpas_pool_get_array(ecosysAuxiliary, 'avgCO2_gas_flux', avgCO2_gas_flux) + real (kind=RKIND) :: surfaceFreezingTemp - end if + logical, pointer :: frazilIceActive, & + config_remove_ais_river_runoff, & + config_remove_ais_ice_runoff, & + config_use_ecosysTracers, & + config_use_DMSTracers, & + config_use_MacroMoleculesTracers, & + config_use_ecosysTracers_sea_ice_coupling, & + config_use_DMSTracers_sea_ice_coupling, & + config_use_MacroMoleculesTracers_sea_ice_coupling - if (config_use_ecosysTracers .and. config_use_ecosysTracers_sea_ice_coupling) then - call mpas_pool_get_subpool(forcingPool, 'ecosysSeaIceCoupling', ecosysSeaIceCoupling) + character (len=StrKIND), pointer :: config_land_ice_flux_mode - call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfacePhytoC', avgOceanSurfacePhytoC) - call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceDIC', avgOceanSurfaceDIC) - call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceNO3', avgOceanSurfaceNO3) - call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceSiO3', avgOceanSurfaceSiO3) - call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceNH4', avgOceanSurfaceNH4) - call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceDOCr', avgOceanSurfaceDOCr) - call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceDOCSemiLabile', avgOceanSurfaceDOCSemiLabile) - call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceFeParticulate', avgOceanSurfaceFeParticulate) - call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceFeDissolved', avgOceanSurfaceFeDissolved) - endif - if (config_use_DMSTracers .and. config_use_DMSTracers_sea_ice_coupling) then - call mpas_pool_get_subpool(forcingPool, 'DMSSeaIceCoupling', DMSSeaIceCoupling) + logical :: keepFrazil - call mpas_pool_get_array(DMSSeaIceCoupling, 'avgOceanSurfaceDMS', avgOceanSurfaceDMS) - call mpas_pool_get_array(DMSSeaIceCoupling, 'avgOceanSurfaceDMSP', avgOceanSurfaceDMSP) - endif - if (config_use_MacroMoleculesTracers .and. config_use_MacroMoleculesTracers_sea_ice_coupling) then - call mpas_pool_get_subpool(forcingPool, 'MacroMoleculesSeaIceCoupling', MacroMoleculesSeaIceCoupling) - call mpas_pool_get_array(MacroMoleculesSeaIceCoupling, 'avgOceanSurfaceDOC', avgOceanSurfaceDOC) - call mpas_pool_get_array(MacroMoleculesSeaIceCoupling, 'avgOceanSurfaceDON', avgOceanSurfaceDON) - endif - ! call mpas_pool_get_array(forcingPool, 'CO2Flux', CO2Flux) - ! call mpas_pool_get_array(forcingPool, 'DMSFlux', DMSFlux) - ! call mpas_pool_get_array(forcingPool, 'surfaceUpwardCO2Flux', surfaceUpwardCO2Flux) + ! get configure options + call mpas_pool_get_package(domain % packages, 'frazilIceActive', frazilIceActive) + call mpas_pool_get_config(domain % configs, 'config_use_ecosysTracers', config_use_ecosysTracers) + call mpas_pool_get_config(domain % configs, 'config_land_ice_flux_mode', config_land_ice_flux_mode) + call mpas_pool_get_config(domain % configs, 'config_remove_ais_river_runoff', config_remove_ais_river_runoff) + call mpas_pool_get_config(domain % configs, 'config_remove_ais_ice_runoff', config_remove_ais_ice_runoff) + call mpas_pool_get_config(domain % configs, 'config_use_DMSTracers', config_use_DMSTracers) + call mpas_pool_get_config(domain % configs, 'config_use_MacroMoleculesTracers', config_use_MacroMoleculesTracers) + call mpas_pool_get_config(domain % configs, 'config_use_ecosysTracers_sea_ice_coupling', & + config_use_ecosysTracers_sea_ice_coupling) + call mpas_pool_get_config(domain % configs, 'config_use_DMSTracers_sea_ice_coupling', & + config_use_DMSTracers_sea_ice_coupling) + call mpas_pool_get_config(domain % configs, 'config_use_MacroMoleculesTracers_sea_ice_coupling', & + config_use_MacroMoleculesTracers_sea_ice_coupling) + + n = 0 + block_ptr => domain % blocklist + do while(associated(block_ptr)) + call mpas_pool_get_subpool(block_ptr % structs, 'mesh', meshPool) + call mpas_pool_get_subpool(block_ptr % structs, 'forcing', forcingPool) + call mpas_pool_get_subpool(block_ptr % structs, 'state', statePool) - do i = 1, nCellsSolve - n = n + 1 + call mpas_pool_get_subpool(statePool, 'tracers', tracersPool) - o2x_om(n, index_o2x_So_t) = avgTracersSurfaceValue(index_temperatureSurfaceValue, i) - o2x_om(n, index_o2x_So_s) = avgTracersSurfaceValue(index_salinitySurfaceValue, i) - o2x_om(n, index_o2x_So_u) = avgSurfaceVelocity(index_avgZonalSurfaceVelocity, i) - o2x_om(n, index_o2x_So_v) = avgSurfaceVelocity(index_avgMeridionalSurfaceVelocity, i) + call mpas_pool_get_dimension(meshPool, 'nCellsSolve', nCellsSolve) - o2x_om(n, index_o2x_So_ssh) = ssh(i) - o2x_om(n, index_o2x_So_dhdx) = avgSSHGradient(index_avgZonalSSHGradient, i) - o2x_om(n, index_o2x_So_dhdy) = avgSSHGradient(index_avgMeridionalSSHGradient, i) + call mpas_pool_get_dimension(forcingPool, 'index_avgTemperatureSurfaceValue', index_temperatureSurfaceValue) + call mpas_pool_get_dimension(forcingPool, 'index_avgSalinitySurfaceValue', index_salinitySurfaceValue) + call mpas_pool_get_dimension(forcingPool, 'index_avgSurfaceVelocityZonal', index_avgZonalSurfaceVelocity) + call mpas_pool_get_dimension(forcingPool, 'index_avgSurfaceVelocityMeridional', index_avgMeridionalSurfaceVelocity) + call mpas_pool_get_dimension(forcingPool, 'index_avgSSHGradientZonal', index_avgZonalSSHGradient) + call mpas_pool_get_dimension(forcingPool, 'index_avgSSHGradientMeridional', index_avgMeridionalSSHGradient) - o2x_om(n, index_o2x_Faoo_h2otemp) = avgTotalFreshWaterTemperatureFlux(i) * rho_sw * cp_sw + call mpas_pool_get_array(statePool, 'ssh', ssh, 1) + call mpas_pool_get_array(statePool, 'layerThickness', layerThickness, 1) - if (trim(config_land_ice_flux_mode) == 'standalone' .or. trim(config_land_ice_flux_mode) == 'data') then - o2x_om(n, index_o2x_Foxo_ismw) = avgLandIceFreshwaterFlux(i) - o2x_om(n, index_o2x_Foxo_ismh) = avgLandIceHeatFlux(i) - endif - if (config_remove_ais_river_runoff) then - o2x_om(n, index_o2x_Foxo_rrofl) = avgRemovedRiverRunoffFlux(i) - endif - if (config_remove_ais_ice_runoff) then - o2x_om(n, index_o2x_Foxo_rrofi) = avgRemovedIceRunoffFlux(i) - o2x_om(n, index_o2x_Foxo_rrofih) = avgRemovedIceRunoffHeatFlux(i) - endif + call mpas_pool_get_array(forcingPool, 'landIceMask', landIceMask) + call mpas_pool_get_array(forcingPool, 'avgTracersSurfaceValue', avgTracersSurfaceValue) + call mpas_pool_get_array(forcingPool, 'avgSurfaceVelocity', avgSurfaceVelocity) + call mpas_pool_get_array(forcingPool, 'avgSSHGradient', avgSSHGradient) + call mpas_pool_get_array(forcingPool, 'avgTotalFreshWaterTemperatureFlux', avgTotalFreshWaterTemperatureFlux) - if ( frazilIceActive ) then - ! negative when frazil ice can be melted - keepFrazil = .true. - if ( associated(landIceMask) ) then - if ( landIceMask(i) == 1 ) then - keepFrazil = .false. - end if - end if + if ( frazilIceActive ) then + call mpas_pool_get_array(forcingPool, 'seaIceEnergy', seaIceEnergy) + call mpas_pool_get_array(forcingPool, 'frazilSurfacePressure', frazilSurfacePressure) + call mpas_pool_get_array(statePool, 'accumulatedFrazilIceMass', accumulatedFrazilIceMass, 1) + end if - if ( keepFrazil ) then + ! Cryo fields + if (trim(config_land_ice_flux_mode) == 'standalone' .or. trim(config_land_ice_flux_mode) == 'data') then + call mpas_pool_get_array(forcingPool, 'avgLandIceFreshwaterFlux', avgLandIceFreshwaterFlux) + call mpas_pool_get_array(forcingPool, 'avgLandIceHeatFlux', avgLandIceHeatFlux) + endif + if (config_remove_ais_river_runoff) then + call mpas_pool_get_array(forcingPool, 'avgRemovedRiverRunoffFlux', avgRemovedRiverRunoffFlux) + endif + if (config_remove_ais_ice_runoff) then + call mpas_pool_get_array(forcingPool, 'avgRemovedIceRunoffFlux', avgRemovedIceRunoffFlux) + call mpas_pool_get_array(forcingPool, 'avgRemovedIceRunoffHeatFlux', avgRemovedIceRunoffHeatFlux) + endif - ! Calculate energy associated with frazil mass transfer to sea ice if frazil has accumulated - if ( accumulatedFrazilIceMass(i) > 0.0_RKIND ) then + ! BGC fields + if (config_use_ecosysTracers) then - seaIceEnergy(i) = accumulatedFrazilIceMass(i) * config_frazil_heat_of_fusion + call mpas_pool_get_subpool(forcingPool, 'ecosysAuxiliary', ecosysAuxiliary) + call mpas_pool_get_array(ecosysAuxiliary, 'avgCO2_gas_flux', avgCO2_gas_flux) - ! Otherwise calculate the melt potential where avgTracersSurfaceValue represents only the - ! top layer of the ocean - else + end if - surfaceFreezingTemp = ocn_freezing_temperature(salinity=avgTracersSurfaceValue(index_salinitySurfaceValue, i), & - pressure=0.0_RKIND, inLandIceCavity=.false.) + if (config_use_ecosysTracers .and. config_use_ecosysTracers_sea_ice_coupling) then + call mpas_pool_get_subpool(forcingPool, 'ecosysSeaIceCoupling', ecosysSeaIceCoupling) - seaIceEnergy(i) = min(rho_sw*cp_sw*layerThickness(1, i)*( surfaceFreezingTemp + T0_Kelvin & - - avgTracersSurfaceValue(index_temperatureSurfaceValue, i) ), 0.0_RKIND ) + call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfacePhytoC', avgOceanSurfacePhytoC) + call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceDIC', avgOceanSurfaceDIC) + call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceNO3', avgOceanSurfaceNO3) + call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceSiO3', avgOceanSurfaceSiO3) + call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceNH4', avgOceanSurfaceNH4) + call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceDOCr', avgOceanSurfaceDOCr) + call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceDOCSemiLabile', avgOceanSurfaceDOCSemiLabile) + call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceFeParticulate', avgOceanSurfaceFeParticulate) + call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceFeDissolved', avgOceanSurfaceFeDissolved) + endif + if (config_use_DMSTracers .and. config_use_DMSTracers_sea_ice_coupling) then + call mpas_pool_get_subpool(forcingPool, 'DMSSeaIceCoupling', DMSSeaIceCoupling) - end if + call mpas_pool_get_array(DMSSeaIceCoupling, 'avgOceanSurfaceDMS', avgOceanSurfaceDMS) + call mpas_pool_get_array(DMSSeaIceCoupling, 'avgOceanSurfaceDMSP', avgOceanSurfaceDMSP) + endif + if (config_use_MacroMoleculesTracers .and. config_use_MacroMoleculesTracers_sea_ice_coupling) then + call mpas_pool_get_subpool(forcingPool, 'MacroMoleculesSeaIceCoupling', MacroMoleculesSeaIceCoupling) - o2x_om(n, index_o2x_Fioo_q) = seaIceEnergy(i) / ocn_cpl_dt - o2x_om(n, index_o2x_Fioo_frazil) = accumulatedFrazilIceMass(i) / ocn_cpl_dt + call mpas_pool_get_array(MacroMoleculesSeaIceCoupling, 'avgOceanSurfaceDOC', avgOceanSurfaceDOC) + call mpas_pool_get_array(MacroMoleculesSeaIceCoupling, 'avgOceanSurfaceDON', avgOceanSurfaceDON) + endif +! call mpas_pool_get_array(forcingPool, 'CO2Flux', CO2Flux) +! call mpas_pool_get_array(forcingPool, 'DMSFlux', DMSFlux) +! call mpas_pool_get_array(forcingPool, 'surfaceUpwardCO2Flux', surfaceUpwardCO2Flux) - else +! replace 'o2x_o % rAttr(' with 'o2x_om(n, ' and ', n)' with ')' + do i = 1, nCellsSolve + n = n + 1 - o2x_om(n, index_o2x_Fioo_q) = 0.0_RKIND - o2x_om(n, index_o2x_Fioo_frazil) = 0.0_RKIND - if (trim(config_land_ice_flux_mode) == 'standalone' .or. trim(config_land_ice_flux_mode) == 'data') then - o2x_om(n, index_o2x_Foxo_q_li) = accumulatedFrazilIceMass(i) * config_frazil_heat_of_fusion / ocn_cpl_dt - o2x_om(n, index_o2x_Foxo_frazil_li) = accumulatedFrazilIceMass(i) / ocn_cpl_dt - endif - end if + o2x_om(n, index_o2x_So_t) = avgTracersSurfaceValue(index_temperatureSurfaceValue, i) + o2x_om(n, index_o2x_So_s) = avgTracersSurfaceValue(index_salinitySurfaceValue, i) + o2x_om(n, index_o2x_So_u) = avgSurfaceVelocity(index_avgZonalSurfaceVelocity, i) + o2x_om(n, index_o2x_So_v) = avgSurfaceVelocity(index_avgMeridionalSurfaceVelocity, i) - ! Reset SeaIce Energy and Accumulated Frazil Ice - seaIceEnergy(i) = 0.0_RKIND - accumulatedFrazilIceMass(i) = 0.0_RKIND - frazilSurfacePressure(i) = 0.0_RKIND - end if + o2x_om(n, index_o2x_So_ssh) = ssh(i) + o2x_om(n, index_o2x_So_dhdx) = avgSSHGradient(index_avgZonalSSHGradient, i) + o2x_om(n, index_o2x_So_dhdy) = avgSSHGradient(index_avgMeridionalSSHGradient, i) - ! BGC fields - if (config_use_ecosysTracers) then - ! convert from mmolC/m2/s to kg CO2/m2/s - o2x_om(n, index_o2x_Faoo_fco2_ocn) = avgCO2_gas_flux(i)*44.e-6_RKIND - endif - if (config_use_ecosysTracers .and. config_use_ecosysTracers_sea_ice_coupling) then - o2x_om(n, index_o2x_So_algae1) = max(0.0_RKIND,avgOceanSurfacePhytoC(1,i)) - o2x_om(n, index_o2x_So_algae2) = max(0.0_RKIND,avgOceanSurfacePhytoC(2,i)) - o2x_om(n, index_o2x_So_algae3) = max(0.0_RKIND,avgOceanSurfacePhytoC(3,i)) - o2x_om(n, index_o2x_So_dic1) = max(0.0_RKIND,avgOceanSurfaceDIC(i)) - o2x_om(n, index_o2x_So_doc1) = max(0.0_RKIND,avgOceanSurfaceDOCSemiLabile(i)) - o2x_om(n, index_o2x_So_doc2) = max(0.0_RKIND,avgOceanSurfaceDOCSemiLabile(i)) - o2x_om(n, index_o2x_So_doc3) = max(0.0_RKIND,avgOceanSurfaceDOCSemiLabile(i)) - o2x_om(n, index_o2x_So_don1) = 0.0_RKIND - o2x_om(n, index_o2x_So_no3) = max(0.0_RKIND,avgOceanSurfaceNO3(i)) - o2x_om(n, index_o2x_So_sio3) = max(0.0_RKIND,avgOceanSurfaceSiO3(i)) - o2x_om(n, index_o2x_So_nh4) = max(0.0_RKIND,avgOceanSurfaceNH4(i)) - o2x_om(n, index_o2x_So_docr) = max(0.0_RKIND,avgOceanSurfaceDOCr(i)) - o2x_om(n, index_o2x_So_fep1) = max(0.0_RKIND,avgOceanSurfaceFeParticulate(i)) - o2x_om(n, index_o2x_So_fed1) = max(0.0_RKIND,avgOceanSurfaceFeDissolved(i)) - endif - if (config_use_DMSTracers .and. config_use_DMSTracers_sea_ice_coupling) then - o2x_om(n, index_o2x_So_dms) = max(0.0_RKIND,avgOceanSurfaceDMS(i)) - o2x_om(n, index_o2x_So_dmsp) = max(0.0_RKIND,avgOceanSurfaceDMSP(i)) - endif - if (config_use_MacroMoleculesTracers .and. config_use_MacroMoleculesTracers_sea_ice_coupling) then - o2x_om(n, index_o2x_So_doc1) = max(0.0_RKIND,avgOceanSurfaceDOC(1,i)) - o2x_om(n, index_o2x_So_doc2) = max(0.0_RKIND,avgOceanSurfaceDOC(2,i)) - o2x_om(n, index_o2x_So_don1) = max(0.0_RKIND,avgOceanSurfaceDON(i)) - endif + o2x_om(n, index_o2x_Faoo_h2otemp) = avgTotalFreshWaterTemperatureFlux(i) * rho_sw * cp_sw - if ( trim(config_land_ice_flux_mode) .eq. 'standalone' .or. & - trim(config_land_ice_flux_mode) .eq. 'coupled' ) then - o2x_om(n, index_o2x_So_blt) = landIceBoundaryLayerTracers(indexBLT,i) - o2x_om(n, index_o2x_So_bls) = landIceBoundaryLayerTracers(indexBLS,i) - o2x_om(n, index_o2x_So_htv) = landIceTracerTransferVelocities(indexHeatTrans,i) - o2x_om(n, index_o2x_So_stv) = landIceTracerTransferVelocities(indexSaltTrans,i) - o2x_om(n, index_o2x_So_rhoeff) = 0.0_RKIND - endif - end do + ! Cryo fields + if (trim(config_land_ice_flux_mode) == 'standalone' .or. trim(config_land_ice_flux_mode) == 'data') then + o2x_om(n, index_o2x_Foxo_ismw) = avgLandIceFreshwaterFlux(i) + o2x_om(n, index_o2x_Foxo_ismh) = avgLandIceHeatFlux(i) + endif + if (config_remove_ais_river_runoff) then + o2x_om(n, index_o2x_Foxo_rrofl) = avgRemovedRiverRunoffFlux(i) + endif + if (config_remove_ais_ice_runoff) then + o2x_om(n, index_o2x_Foxo_rrofi) = avgRemovedIceRunoffFlux(i) + o2x_om(n, index_o2x_Foxo_rrofih) = avgRemovedIceRunoffHeatFlux(i) + endif - block_ptr => block_ptr % next - end do + if ( frazilIceActive ) then + ! negative when frazil ice can be melted + keepFrazil = .true. + if ( associated(landIceMask) ) then + if ( landIceMask(i) == 1 ) then + keepFrazil = .false. + end if + end if + + if ( keepFrazil ) then + + ! Calculate energy associated with frazil mass transfer to sea ice if frazil has accumulated + if ( accumulatedFrazilIceMass(i) > 0.0_RKIND ) then + + seaIceEnergy(i) = accumulatedFrazilIceMass(i) * config_frazil_heat_of_fusion + + ! Otherwise calculate the melt potential where avgTracersSurfaceValue represents only the + ! top layer of the ocean + else + + surfaceFreezingTemp = ocn_freezing_temperature(salinity=avgTracersSurfaceValue(index_salinitySurfaceValue, i), & + pressure=0.0_RKIND, inLandIceCavity=.false.) + + seaIceEnergy(i) = min(rho_sw*cp_sw*layerThickness(1, i)*( surfaceFreezingTemp + T0_Kelvin & + - avgTracersSurfaceValue(index_temperatureSurfaceValue, i) ), 0.0_RKIND ) + + end if + + o2x_om(n, index_o2x_Fioo_q) = seaIceEnergy(i) / ocn_cpl_dt + o2x_om(n, index_o2x_Fioo_frazil) = accumulatedFrazilIceMass(i) / ocn_cpl_dt + + else + + o2x_om(n, index_o2x_Fioo_q) = 0.0_RKIND + o2x_om(n, index_o2x_Fioo_frazil) = 0.0_RKIND + if (trim(config_land_ice_flux_mode) == 'standalone' .or. trim(config_land_ice_flux_mode) == 'data') then + o2x_om(n, index_o2x_Foxo_q_li) = accumulatedFrazilIceMass(i) * config_frazil_heat_of_fusion / ocn_cpl_dt + o2x_om(n, index_o2x_Foxo_frazil_li) = accumulatedFrazilIceMass(i) / ocn_cpl_dt + endif + + end if + + ! Reset SeaIce Energy and Accumulated Frazil Ice + seaIceEnergy(i) = 0.0_RKIND + accumulatedFrazilIceMass(i) = 0.0_RKIND + frazilSurfacePressure(i) = 0.0_RKIND + end if + + ! BGC fields + if (config_use_ecosysTracers .and. index_o2x_Faoo_fco2_ocn /= 0) then + ! convert from mmolC/m2/s to kg CO2/m2/s + o2x_om(n, index_o2x_Faoo_fco2_ocn) = avgCO2_gas_flux(i)*44.e-6_RKIND + endif + if (config_use_ecosysTracers .and. config_use_ecosysTracers_sea_ice_coupling) then + o2x_om(n, index_o2x_So_algae1) = max(0.0_RKIND,avgOceanSurfacePhytoC(1,i)) + o2x_om(n, index_o2x_So_algae2) = max(0.0_RKIND,avgOceanSurfacePhytoC(2,i)) + o2x_om(n, index_o2x_So_algae3) = max(0.0_RKIND,avgOceanSurfacePhytoC(3,i)) + o2x_om(n, index_o2x_So_dic1) = max(0.0_RKIND,avgOceanSurfaceDIC(i)) + o2x_om(n, index_o2x_So_doc1) = max(0.0_RKIND,avgOceanSurfaceDOCSemiLabile(i)) + o2x_om(n, index_o2x_So_doc2) = max(0.0_RKIND,avgOceanSurfaceDOCSemiLabile(i)) + o2x_om(n, index_o2x_So_doc3) = max(0.0_RKIND,avgOceanSurfaceDOCSemiLabile(i)) + o2x_om(n, index_o2x_So_don1) = 0.0_RKIND + o2x_om(n, index_o2x_So_no3) = max(0.0_RKIND,avgOceanSurfaceNO3(i)) + o2x_om(n, index_o2x_So_sio3) = max(0.0_RKIND,avgOceanSurfaceSiO3(i)) + o2x_om(n, index_o2x_So_nh4) = max(0.0_RKIND,avgOceanSurfaceNH4(i)) + o2x_om(n, index_o2x_So_docr) = max(0.0_RKIND,avgOceanSurfaceDOCr(i)) + o2x_om(n, index_o2x_So_fep1) = max(0.0_RKIND,avgOceanSurfaceFeParticulate(i)) + o2x_om(n, index_o2x_So_fed1) = max(0.0_RKIND,avgOceanSurfaceFeDissolved(i)) + endif + if (config_use_DMSTracers .and. config_use_DMSTracers_sea_ice_coupling) then + o2x_om(n, index_o2x_So_dms) = max(0.0_RKIND,avgOceanSurfaceDMS(i)) + o2x_om(n, index_o2x_So_dmsp) = max(0.0_RKIND,avgOceanSurfaceDMSP(i)) + endif + if (config_use_MacroMoleculesTracers .and. config_use_MacroMoleculesTracers_sea_ice_coupling) then + o2x_om(n, index_o2x_So_doc1) = max(0.0_RKIND,avgOceanSurfaceDOC(1,i)) + o2x_om(n, index_o2x_So_doc2) = max(0.0_RKIND,avgOceanSurfaceDOC(2,i)) + o2x_om(n, index_o2x_So_doc3) = max(0.0_RKIND,avgOceanSurfaceDOC(3,i)) + o2x_om(n, index_o2x_So_don1) = 0.0_RKIND + endif +! o2x_om(n, index_o2x_Faoo_fco2_ocn) = CO2Flux(i) +! o2x_om(n, index_o2x_Faoo_fdms_ocn) = DMSFlux(i) +! o2x_om(n, index_o2x_Faoo_fco2_ocn) = surfaceUpwardCO2Flux(i) + +!JW o2x_om(n, index_o2x_So_blt) = landIceBoundaryLayerTemperature(i) +!JW o2x_om(n, index_o2x_So_bls) = landIceBoundaryLayerSalinity(i) +!JW o2x_om(n, index_o2x_So_htv) = landIceHeatTransferVelocity(i) +!JW o2x_om(n, index_o2x_So_stv) = landIceSaltTransferVelocity(i) + + if ( trim(config_land_ice_flux_mode) .eq. 'standalone' .or. & + trim(config_land_ice_flux_mode) .eq. 'coupled' ) then + o2x_om(n, index_o2x_So_blt) = landIceBoundaryLayerTracers(indexBLT,i) + o2x_om(n, index_o2x_So_bls) = landIceBoundaryLayerTracers(indexBLS,i) + o2x_om(n, index_o2x_So_htv) = landIceTracerTransferVelocities(indexHeatTrans,i) + o2x_om(n, index_o2x_So_stv) = landIceTracerTransferVelocities(indexSaltTrans,i) + o2x_om(n, index_o2x_So_rhoeff) = 0.0_RKIND + endif + + !Fyke: test + !write(stderrUnit,*) 'n=',n + !write(stderrUnit,*) 'o2x_om(n, index_o2x_So_blt)=',o2x_om(n, index_o2x_So_blt) + !write(stderrUnit,*) 'o2x_om(n, index_o2x_So_bls)=',o2x_om(n, index_o2x_So_bls) + !write(stderrUnit,*) 'o2x_om(n, index_o2x_So_htv)=',o2x_om(n, index_o2x_So_htv) + !write(stderrUnit,*) 'o2x_om(n, index_o2x_So_stv)=',o2x_om(n, index_o2x_So_stv) + !write(stderrUnit,*) 'o2x_om(n, index_o2x_So_rhoeff)=',o2x_om(n, index_o2x_So_rhoeff) + !o2x_om(n, index_o2x_So_blt) = 0._r8 + !o2x_om(n, index_o2x_So_bls) = 34.5_r8 + !o2x_om(n, index_o2x_So_htv) = 1.e-4_r8 + !o2x_om(n, index_o2x_So_stv) = 3.e-6_r8 + !o2x_om(n, index_o2x_So_rhoeff) = 1000._r8*9.81_r8*918._r8 !lithostatic pressure of 1km of ice + + end do + block_ptr => block_ptr % next + end do ent_type = 1 ! cells ! set all tags in one method tagname = trim(seq_flds_o2x_fields)//C_NULL_CHAR @@ -4251,9 +4571,12 @@ subroutine ocn_export_moab(EClock) !{{{ wopts = 'PARALLEL=WRITE_PART'//C_NULL_CHAR ierr = iMOAB_WriteMesh(MPOID, outfile, wopts) #endif - end subroutine ocn_export_moab!}}} -#endif +!----------------------------------------------------------------------- +!EOC + + end subroutine ocn_export_moab!}}} +#endif end module ocn_comp_mct !||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| diff --git a/components/mpas-ocean/src/Registry.xml b/components/mpas-ocean/src/Registry.xml index 2a21e22f40d2..87100a22a860 100644 --- a/components/mpas-ocean/src/Registry.xml +++ b/components/mpas-ocean/src/Registry.xml @@ -113,6 +113,9 @@ + @@ -807,6 +810,22 @@ description="If true, solid runoff from the Antarctic Ice Sheet (below 60S latitude) coming from the coupled is zeroed in the coupler import routines. To be used with data iceberg fluxes coming from the sea ice model." possible_values=".true. or .false." /> + + + + - @@ -1717,6 +1736,7 @@ + @@ -2008,6 +2028,7 @@ + @@ -2043,6 +2064,11 @@ + + + + + + + + + + + @@ -4010,6 +4060,9 @@ description="The time-averaged effective ocean density within ice shelves based on Archimedes' principle." packages="landIceCouplingPKG" /> + + + + + @@ -304,12 +307,12 @@ + - diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F index beed937d6767..95a9bca8f70c 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F @@ -19,6 +19,9 @@ module ocn_conservation_check + use shr_kind_mod, only: SHR_KIND_R8 + use shr_const_mod + use mpas_derived_types use mpas_pool_routines use mpas_dmpar @@ -158,10 +161,9 @@ subroutine ocn_init_conservation_check(domain, err)!{{{ !----------------------------------------------------------------- ! taking PI from SHR_CONST_PI in share/util/shr_const_mod.F90 to match coupler - !real (kind=RKIND), parameter :: piE3SM = 3.14159265358979323846_RKIND ! pi - real (kind=RKIND), parameter :: piE3SM = 3.141592653589793_RKIND + real (kind=RKIND), parameter :: piE3SM = SHR_CONST_PI ! taking earth radius from SHR_CONST_REARTH in share/util/shr_const_mod.F90 to match coupler - real (kind=RKIND), parameter :: earthRadiusE3SM = 6.371229e6_RKIND ! radius of earth, m + real (kind=RKIND), parameter :: earthRadiusE3SM = SHR_CONST_REARTH ! radius of earth, m err = 0 diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_global_stats.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_global_stats.F index 203665999a86..45e5afdc2448 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_global_stats.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_global_stats.F @@ -243,7 +243,7 @@ subroutine ocn_compute_global_stats(domain, timeLevel, err)!{{{ real (kind=RKIND), pointer :: volumeCellGlobal, volumeEdgeGlobal, CFLNumberGlobal, areaCellGlobal, & areaEdgeGlobal, areaTriangleGlobal, totalVolumeChange, netFreshwaterInput, & absoluteFreshWaterConservation, relativeFreshWaterConservation, & - landIceFloatingAreaSum + landIceFloatingAreaSum, gsRunningMeanRemovedIceRunoff, runningMeanRemovedIceRunoff real (kind=RKIND), dimension(:), pointer :: areaCell, dcEdge, dvEdge, areaTriangle, areaEdge, landIceFloatingArea real (kind=RKIND), dimension(:,:), pointer :: layerThickness, normalVelocity, & lowFreqDivergence, highFreqThickness, & @@ -1385,6 +1385,14 @@ subroutine ocn_compute_global_stats(domain, timeLevel, err)!{{{ rms(variableIndex) = 0.0_RKIND end if + if (config_scale_dismf_by_removed_ice_runoff) then + block => domain % blocklist + call mpas_pool_get_subpool(block % structs, 'forcing', forcingPool) + call mpas_pool_get_array(globalStatsAMPool, 'gsRunningMeanRemovedIceRunoff', gsRunningMeanRemovedIceRunoff) + call mpas_pool_get_array(forcingPool, 'runningMeanRemovedIceRunoff', runningMeanRemovedIceRunoff) + gsRunningMeanRemovedIceRunoff = runningMeanRemovedIceRunoff + end if + ! calculate fresh water conservation check quantities absoluteFreshWaterConservation = totalVolumeChange - netFreshwaterInput if (abs(totalVolumeChange) < 1e-12_RKIND) then diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_harmonic_analysis.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_harmonic_analysis.F index ee1495a3e59a..c60ac179ccf4 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_harmonic_analysis.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_harmonic_analysis.F @@ -681,7 +681,7 @@ SUBROUTINE harmonic_analysis_solve(MNP,nfreq,hmat,GLOELV,haff,haface,emagt,phase REAL(kind=RKIND),ALLOCATABLE :: hap(:),hax(:) REAL(kind=RKIND),ALLOCATABLE :: ha(:,:) - convrd=180.0_RKIND/pii + convrd=180.0_RKIND/pi mm = 2*nfreq ALLOCATE ( PHASEE(nfreq),EMAG(nfreq) ) diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_high_frequency_output.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_high_frequency_output.F index 60a5dececcb3..8ed030392cb8 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_high_frequency_output.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_high_frequency_output.F @@ -175,7 +175,7 @@ subroutine ocn_compute_high_frequency_output(domain, timeLevel, err)!{{{ type (mpas_pool_type), pointer :: highFrequencyOutputAMPool type (mpas_pool_type), pointer :: tracersPool - integer :: iLevel, iLevelTarget, iCell, iEdge, i, cell1, cell2, k, eoe + integer :: iLevel, iCell, iEdge, i, cell1, cell2, k, eoe integer :: iLevel0100, iLevel0250, iLevel0700, iLevel2000 real (kind=RKIND) :: sumLayerThickness integer, pointer :: nVertLevels, nCells, nEdges @@ -183,7 +183,7 @@ subroutine ocn_compute_high_frequency_output(domain, timeLevel, err)!{{{ integer, dimension(:,:), pointer :: edgesOnCell, cellsOnEdge, edgesOnEdge real (kind=RKIND) :: invAreaCell1, layerThicknessEdge1, coeff, weightedNormalVel, cellArea - real (kind=RKIND), dimension(:), pointer :: refBottomDepth, kineticEnergyAt250m, kineticEnergyAtSurface, relativeVorticityAt250m + real (kind=RKIND), dimension(:), pointer :: refBottomDepth, kineticEnergyAt250m, kineticEnergyAtSurface, relativeVorticityAt250m, relativeVorticityAtSurface real (kind=RKIND), dimension(:), pointer :: divergenceAt250m, relativeVorticityVertexAt250m real (kind=RKIND), dimension(:), pointer :: divergenceAtBottom,relativeVorticityAtBottom,kineticEnergyAtBottom real (kind=RKIND), dimension(:), pointer :: vertVelAt250m @@ -263,6 +263,7 @@ subroutine ocn_compute_high_frequency_output(domain, timeLevel, err)!{{{ call mpas_pool_get_array(highFrequencyOutputAMPool, 'kineticEnergyAt250m', kineticEnergyAt250m) call mpas_pool_get_array(highFrequencyOutputAMPool, 'kineticEnergyAtSurface', kineticEnergyAtSurface) call mpas_pool_get_array(highFrequencyOutputAMPool, 'relativeVorticityAt250m', relativeVorticityAt250m) + call mpas_pool_get_array(highFrequencyOutputAMPool, 'relativeVorticityAtSurface', relativeVorticityAtSurface) call mpas_pool_get_array(highFrequencyOutputAMPool, 'divergenceAt250m', divergenceAt250m) call mpas_pool_get_array(highFrequencyOutputAMPool, 'relativeVorticityAtBottom', relativeVorticityAtBottom) call mpas_pool_get_array(highFrequencyOutputAMPool, 'divergenceAtBottom', divergenceAtBottom) @@ -377,37 +378,41 @@ subroutine ocn_compute_high_frequency_output(domain, timeLevel, err)!{{{ call mpas_pool_get_array(highFrequencyOutputAMPool, 'columnIntegratedSpeed', columnIntegratedSpeed) ! find vertical level that is just above the 100 m reference level - iLevel0100 = 1 - do iLevel=2,nVertLevels + ! if even the bottom level isn't deep enough, we still default to the bottom level + iLevel0100 = nVertLevels + do iLevel=1,nVertLevels if(refBottomDepth(iLevel) > 100.0_RKIND) then - iLevel0100 = iLevel-1 + iLevel0100 = iLevel exit endif enddo ! find vertical level that is just above the 250 m reference level - iLevel0250 = 1 + ! if even the bottom level isn't deep enough, we still default to the bottom level + iLevel0250 = nVertLevels do iLevel=iLevel0100,nVertLevels if(refBottomDepth(iLevel) > 250.0_RKIND) then - iLevel0250 = iLevel-1 + iLevel0250 = iLevel exit endif enddo ! find vertical level that is just above the 700 m reference level - iLevel0700 = 1 + ! if even the bottom level isn't deep enough, we still default to the bottom level + iLevel0700 = nVertLevels do iLevel=iLevel0250,nVertLevels if(refBottomDepth(iLevel) > 700.0_RKIND) then - iLevel0700 = iLevel-1 + iLevel0700 = iLevel exit endif enddo ! find vertical level that is just above the 2000 m reference level - iLevel2000 = 1 + ! if even the bottom level isn't deep enough, we still default to the bottom level + iLevel2000 = nVertLevels do iLevel=iLevel0700,nVertLevels if(refBottomDepth(iLevel) > 2000.0_RKIND) then - iLevel2000 = iLevel-1 + iLevel2000 = iLevel exit endif enddo @@ -418,6 +423,7 @@ subroutine ocn_compute_high_frequency_output(domain, timeLevel, err)!{{{ divergenceAt250m(:) = divergence(iLevel0250,:) relativeVorticityVertexAt250m(:) = relativeVorticity(iLevel0250,:) kineticEnergyAtSurface(:) = kineticEnergyCell(1,:) + relativeVorticityAtSurface(:) = relativeVorticityCell(1,:) activeTracersAtSurface(1,:) = activeTracers(1,1,:) activeTracersAtSurface(2,:) = activeTracers(2,1,:) activeTracersAt250m(1,:) = activeTracers(1,iLevel0250,:) diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_lagrangian_particle_tracking.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_lagrangian_particle_tracking.F index ba02542eb45d..8a8a2665bcb9 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_lagrangian_particle_tracking.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_lagrangian_particle_tracking.F @@ -3833,7 +3833,7 @@ subroutine convert_latlon_from_xyz(lat, lon, x, y, z) !{{{ ! ensure range in 0, 2*pi if (lon < 0.0_RKIND) then - lon = 2*pii + lon + lon = 2*pi + lon end if end subroutine convert_latlon_from_xyz !}}} diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_okubo_weiss.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_okubo_weiss.F index 88c9960092b7..64ab02e05733 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_okubo_weiss.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_okubo_weiss.F @@ -946,8 +946,8 @@ subroutine ocn_compute_eddy_stats(dminfo, block, nVertLevels, nCells, nCellsSolv ! for lat/lon coordinates, convert from radians to degrees for output if (config_AM_okuboWeiss_use_lat_lon_coords) then - wsPosX = wsPosX *180.0_RKIND/pii - wsPosY = wsPosY *180.0_RKIND/pii + wsPosX = wsPosX *180.0_RKIND/pi + wsPosY = wsPosY *180.0_RKIND/pi end if call mpas_timer_stop("stats per proc") diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_time_filters.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_time_filters.F index 78adb9a0583f..498f3f09c7ac 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_time_filters.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_time_filters.F @@ -62,7 +62,7 @@ module ocn_time_filters !-------------------------------------------------------------------- #ifdef TIME_FILTERS_DEBUG integer :: iEdgeOutput = 0, iBlockOutput = 0, iklevel = 1 - real (kind=RKIND) :: lonEdgePoint = 10.0_RKIND*pii/180.0_RKIND, latEdgePoint = 30.0_RKIND*pii/180.0_RKIND + real (kind=RKIND) :: lonEdgePoint = 10.0_RKIND*pi/180.0_RKIND, latEdgePoint = 30.0_RKIND*pi/180.0_RKIND #endif !*********************************************************************** @@ -182,7 +182,7 @@ subroutine ocn_init_time_filters(domain, err)!{{{ call mpas_pool_get_subpool(block % structs, 'mesh', statePool) call mpas_pool_get_array(statePool, 'latEdge', latEdge) call mpas_pool_get_array(statePool, 'lonEdge', lonEdge) - print *, 'lon = ', 180.0_RKIND/pii*lonEdge(iEdgeOutput), ' lat = ', 180.0_RKIND/pii*latEdge(iEdgeOutput), & + print *, 'lon = ', 180.0_RKIND/pi*lonEdge(iEdgeOutput), ' lat = ', 180.0_RKIND/pi*latEdge(iEdgeOutput), & ' iklevel=',iklevel, ' iEdgeOutput=',iEdgeOutput, ' iBlockOutput = ', iBlockOutput #endif diff --git a/components/mpas-ocean/src/analysis_members/shr_const_mod.F b/components/mpas-ocean/src/analysis_members/shr_const_mod.F new file mode 120000 index 000000000000..c471e79113fd --- /dev/null +++ b/components/mpas-ocean/src/analysis_members/shr_const_mod.F @@ -0,0 +1 @@ +../../../../share/util/shr_const_mod.F90 \ No newline at end of file diff --git a/components/mpas-ocean/src/analysis_members/shr_kind_mod.F b/components/mpas-ocean/src/analysis_members/shr_kind_mod.F new file mode 120000 index 000000000000..77a61f967b6a --- /dev/null +++ b/components/mpas-ocean/src/analysis_members/shr_kind_mod.F @@ -0,0 +1 @@ +../../../../share/util/shr_kind_mod.F90 \ No newline at end of file diff --git a/components/mpas-ocean/src/driver/mpas_ocn_core_interface.F b/components/mpas-ocean/src/driver/mpas_ocn_core_interface.F index 6d542fb0c3f3..66c2349e1515 100644 --- a/components/mpas-ocean/src/driver/mpas_ocn_core_interface.F +++ b/components/mpas-ocean/src/driver/mpas_ocn_core_interface.F @@ -140,6 +140,7 @@ function ocn_setup_packages(configPool, packagePool, iocontext) result(ierr)!{{{ logical, pointer :: verticalRemapPKGActive logical, pointer :: activeWavePKGActive logical, pointer :: subgridWetDryPKGActive + logical, pointer :: scaledDISMFPKGActive type (mpas_pool_iterator_type) :: pkgItr logical, pointer :: packageActive @@ -175,6 +176,7 @@ function ocn_setup_packages(configPool, packagePool, iocontext) result(ierr)!{{{ logical, pointer :: config_use_gotm logical, pointer :: config_use_active_wave logical, pointer :: config_use_subgrid_wetting_drying + logical, pointer :: config_scale_dismf_by_removed_ice_runoff character (len=StrKIND), pointer :: config_time_integrator character (len=StrKIND), pointer :: config_ocean_run_mode @@ -449,6 +451,17 @@ function ocn_setup_packages(configPool, packagePool, iocontext) result(ierr)!{{{ subgridWetDryPKGActive = .true. end if + ! + ! test for scaling data ice-shelf melt fluxes by the running mean of removed ice runoff + ! + call mpas_pool_get_package(packagePool, 'scaledDISMFPKGActive', scaledDISMFPKGActive) + call mpas_pool_get_config(configPool, & + 'config_scale_dismf_by_removed_ice_runoff', & + config_scale_dismf_by_removed_ice_runoff) + if (config_scale_dismf_by_removed_ice_runoff) then + scaledDISMFPKGActive = .true. + end if + ! ! call into analysis member driver to set analysis member packages ! diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F index d5ba6b08eb6b..b7ec0bed0575 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F @@ -26,6 +26,7 @@ module ocn_forward_mode use mpas_stream_manager use mpas_timekeeping use mpas_dmpar + use mpas_forcing use mpas_timer use mpas_log use mpas_decomp @@ -94,6 +95,7 @@ module ocn_forward_mode use ocn_forcing use ocn_time_varying_forcing + use ocn_framework_forcing use ocn_constants use ocn_config @@ -664,7 +666,12 @@ function ocn_forward_mode_run(domain) result(ierr)!{{{ ! initialize time-varying forcing call ocn_time_varying_forcing_init(domain) - call ocn_time_varying_forcing_get(domain % streamManager, domain, domain % clock) + + ! if not using RK4, calculate time varying forcing terms once per + ! time-step as opposed at each RK substage as implemented in RK4 + if (timeIntegratorChoice /= timeIntRK4) then + call ocn_time_varying_forcing_get(domain % streamManager, domain, domain % clock) + endif ! During integration, time level 1 stores the model state at the beginning of the ! time step, and time level 2 stores the state advanced dt in time by timestep(...) @@ -834,7 +841,17 @@ function ocn_forward_mode_run(domain) result(ierr)!{{{ endif ! read in next time level data required for time-varying forcing - call ocn_time_varying_forcing_get(domain % streamManager, domain, domain % clock) + if (timeIntegratorChoice /= timeIntRK4) then + ! if not using RK4, calculate time varying forcing terms once per + ! time-step as opposed at each RK substage as implemented in RK4 + call ocn_time_varying_forcing_get(domain % streamManager, domain, domain % clock) + else + if (config_use_time_varying_atmospheric_forcing .or. & + config_use_time_varying_land_ice_forcing) then + ! increment forcing clock to next time-step + call mpas_advance_forcing_clock(forcingGroupHead, dt) + endif + endif ! Validate that the state is OK to run with for the next timestep. call ocn_validate_state(domain, timeLevel=1) diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration.F index 68edc9335359..89ab12945144 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration.F @@ -48,6 +48,19 @@ module ocn_time_integration ! !-------------------------------------------------------------------- + ! Enum for selecting different time integrators + integer, public :: timeIntegratorChoice + + integer, public, parameter :: & + timeIntUnknown = 0, &! unknown or undefined + timeIntSplitExplicit = 1, &! split-explicit + timeIntUnsplitExplicit = 2, &! unsplit-explicit + timeIntSemiImplicit = 3, &! Semi-implicit + timeIntRK4 = 4, &! 4th-order Runge-Kutta + timeIntLTS = 5, &! local time-stepping + timeIntFBLTS = 6, &! forward-backward lts + timeIntSplitExplicitAB2 = 7 ! split-explicit AB2 baroclinic + !-------------------------------------------------------------------- ! ! Public member functions @@ -63,18 +76,6 @@ module ocn_time_integration ! !-------------------------------------------------------------------- - ! Enum for selecting different time integrators - integer :: timeIntegratorChoice - - integer, parameter :: & - timeIntUnknown = 0, &! unknown or undefined - timeIntSplitExplicit = 1, &! split-explicit - timeIntUnsplitExplicit = 2, &! unsplit-explicit - timeIntSemiImplicit = 3, &! Semi-implicit - timeIntRK4 = 4, &! 4th-order Runge-Kutta - timeIntLTS = 5, &! local time-stepping - timeIntFBLTS = 6, &! forward-backward lts - timeIntSplitExplicitAB2 = 7 ! split-explicit AB2 baroclinic !*********************************************************************** diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_rk4.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_rk4.F index 5fbb56e6ec1f..9c4cda353dff 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_rk4.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_rk4.F @@ -43,6 +43,7 @@ module ocn_time_integration_rk4 use ocn_effective_density_in_land_ice use ocn_surface_land_ice_fluxes use ocn_transport_tests + use ocn_time_varying_forcing use ocn_subgrid @@ -118,7 +119,7 @@ subroutine ocn_time_integrator_rk4(domain, dt)!{{{ type (mpas_pool_type), pointer :: nextProvisPool, prevProvisPool - real (kind=RKIND), dimension(4) :: rk_weights, rk_substep_weights + real (kind=RKIND), dimension(4) :: rk_weights, rk_substep_weights, forcingTimeIncrementRK4 real (kind=RKIND) :: coef real (kind=RKIND), dimension(:,:), pointer :: & @@ -186,6 +187,7 @@ subroutine ocn_time_integrator_rk4(domain, dt)!{{{ ! Tidal boundary condition logical, pointer :: config_use_tidal_forcing character (len=StrKIND), pointer :: config_tidal_forcing_type + real (kind=RKIND), pointer :: forcingTimeIncrement real (kind=RKIND), dimension(:), pointer :: tidalInputMask, tidalBCValue real (kind=RKIND), dimension(:,:), pointer :: restingThickness real (kind=RKIND) :: totalDepth @@ -233,6 +235,7 @@ subroutine ocn_time_integrator_rk4(domain, dt)!{{{ call mpas_pool_get_subpool(statePool, 'tracers', tracersPool) call mpas_pool_get_subpool(block % structs, 'mesh', meshPool) call mpas_pool_get_subpool(block % structs, 'diagnostics', diagnosticsPool) + call mpas_pool_get_subpool(block % structs, 'forcing', forcingPool) call mpas_pool_get_subpool(block % structs, 'provis_state', provisStatePool) @@ -260,6 +263,9 @@ subroutine ocn_time_integrator_rk4(domain, dt)!{{{ call mpas_pool_get_array(meshPool, 'subgridSshCellTableRange', & subgridSshCellTableRange) + call mpas_pool_get_array(forcingPool, 'forcingTimeIncrement', forcingTimeIncrement) + + forcingTimeIncrement = 0.0_RKIND ! Lower k-loop limit of 1 rather than minLevel* needed in *New = *Cur ! assignments below are needed to maintain bit-for-bit results @@ -343,10 +349,10 @@ subroutine ocn_time_integrator_rk4(domain, dt)!{{{ ! The coefficients of k_j are b_j = (1/6, 1/3, 1/3, 1/6) and are ! initialized here as delta t * b_j: - rk_weights(1) = dt/6. - rk_weights(2) = dt/3. - rk_weights(3) = dt/3. - rk_weights(4) = dt/6. + rk_weights(1) = dt/6.0_RKIND + rk_weights(2) = dt/3.0_RKIND + rk_weights(3) = dt/3.0_RKIND + rk_weights(4) = dt/6.0_RKIND ! The a_j coefficients of h in the computation of k_j are typically written (0, 1/2, 1/2, 1). ! However, in the algorithm below we pre-compute the state for the tendency one iteration early. @@ -355,11 +361,19 @@ subroutine ocn_time_integrator_rk4(domain, dt)!{{{ ! That is why the coefficients of h are one index early in the following, i.e. ! a = (1/2, 1/2, 1) - rk_substep_weights(1) = dt/2. - rk_substep_weights(2) = dt/2. + rk_substep_weights(1) = dt/2.0_RKIND + rk_substep_weights(2) = dt/2.0_RKIND rk_substep_weights(3) = dt rk_substep_weights(4) = dt ! a_4 only used for ALE step, otherwise it is skipped. + ! these are time increments to evaluate the tidal forcing at the + ! intermediate time-steps as required by RK4 + + forcingTimeIncrementRK4(1) = 0.0_RKIND + forcingTimeIncrementRK4(2) = dt/2.0_RKIND + forcingTimeIncrementRK4(3) = dt/2.0_RKIND + forcingTimeIncrementRK4(4) = dt + call mpas_timer_start("RK4-main loop") !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -481,8 +495,17 @@ subroutine ocn_time_integrator_rk4(domain, dt)!{{{ block => domain % blocklist do while (associated(block)) - call ocn_time_integrator_rk4_compute_vel_tends(domain, block, dt, rk_substep_weights(rk_step), domain % dminfo, err ) + call mpas_pool_get_subpool(block % structs, 'forcing', forcingPool) + call mpas_pool_get_array(forcingPool, 'forcingTimeIncrement', forcingTimeIncrement) + forcingTimeIncrement = forcingTimeIncrementRK4(rk_step) + block => block % next + end do + + call ocn_time_varying_forcing_get(domain % streamManager, domain, domain % clock) + block => domain % blocklist + do while (associated(block)) + call ocn_time_integrator_rk4_compute_vel_tends(domain, block, dt, rk_substep_weights(rk_step), domain % dminfo, err ) call ocn_time_integrator_rk4_compute_thick_tends( block, dt, rk_substep_weights(rk_step), err ) block => block % next end do diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_si.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_si.F index 9529fbdc6ea1..77bf1331d3d1 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_si.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_si.F @@ -380,6 +380,8 @@ subroutine ocn_time_integrator_si(domain, dt)!{{{ real (kind=RKIND), dimension(:,:,:), pointer :: activeTracersNew + real (kind=RKIND), pointer :: forcingTimeIncrement + ! Remap variables real (kind=RKIND), dimension(:,:), pointer :: & layerThicknessLagNew @@ -507,6 +509,11 @@ subroutine ocn_time_integrator_si(domain, dt)!{{{ call mpas_pool_get_array(tracersPool, 'activeTracers', activeTracersNew, 2) + call mpas_pool_get_array(forcingPool, 'forcingTimeIncrement', & + forcingTimeIncrement) + + forcingTimeIncrement = 0.0_RKIND + allocate(bottomDepthEdge(nEdgesAll+1)) if (config_transport_tests_flow_id > 0) then @@ -2587,7 +2594,7 @@ subroutine ocn_time_integrator_si(domain, dt)!{{{ ! Reset tracer2 to 2 in top n layers ! in zonal bands, and 1 outside - lat = latCell(iCell)*180./3.1415 + lat = latCell(iCell)*180.0_RKIND/pi if ( lat>-60.0.and.lat<-55.0 & .or.lat>-40.0.and.lat<-35.0 & .or.lat>- 2.5.and.lat< 2.5 & @@ -2604,7 +2611,7 @@ subroutine ocn_time_integrator_si(domain, dt)!{{{ ! Reset tracer3 to 2 in top n layers ! in zonal bands, and 1 outside - lat = latCell(iCell)*180./3.1415 + lat = latCell(iCell)*180.0_RKIND/pi if ( lat>-55.0.and.lat<-50.0 & .or.lat>-35.0.and.lat<-30.0 & .or.lat>-15.0.and.lat<-10.0 & diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F index 653d88198131..5c1069f4e3b4 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F @@ -280,6 +280,8 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ real (kind=RKIND), dimension(:,:,:), pointer :: activeTracersNew + real (kind=RKIND), pointer :: forcingTimeIncrement + ! Remap variables real (kind=RKIND), dimension(:,:), pointer :: & layerThicknessLagNew @@ -398,6 +400,11 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ allocate(baroclinicThickness(nEdgesAll+1)) + call mpas_pool_get_array(forcingPool, 'forcingTimeIncrement', & + forcingTimeIncrement) + + forcingTimeIncrement = 0.0_RKIND + if (config_transport_tests_flow_id > 0) then ! This is a transport test. Write advection velocity from prescribed ! flow field. @@ -2225,7 +2232,7 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ ! Reset tracer2 to 2 in top n layers ! in zonal bands, and 1 outside - lat = latCell(iCell)*180./3.1415 + lat = latCell(iCell)*180.0_RKIND/pi if ( lat>-60.0.and.lat<-55.0 & .or.lat>-40.0.and.lat<-35.0 & .or.lat>- 2.5.and.lat< 2.5 & @@ -2242,7 +2249,7 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ ! Reset tracer3 to 2 in top n layers ! in zonal bands, and 1 outside - lat = latCell(iCell)*180./3.1415 + lat = latCell(iCell)*180.0_RKIND/pi if ( lat>-55.0.and.lat<-50.0 & .or.lat>-35.0.and.lat<-30.0 & .or.lat>-15.0.and.lat<-10.0 & diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split_ab2.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split_ab2.F index c6040aeba1d6..0693d00a72fd 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split_ab2.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split_ab2.F @@ -2405,7 +2405,7 @@ subroutine ocn_time_integrator_split_ab2(domain, dt)!{{{ ! Reset tracer2 to 2 in top n layers ! in zonal bands, and 1 outside - lat = latCell(iCell)*180./3.1415 + lat = latCell(iCell)*180.0_RKIND/pi if ( lat>-60.0.and.lat<-55.0 & .or.lat>-40.0.and.lat<-35.0 & .or.lat>- 2.5.and.lat< 2.5 & @@ -2422,7 +2422,7 @@ subroutine ocn_time_integrator_split_ab2(domain, dt)!{{{ ! Reset tracer3 to 2 in top n layers ! in zonal bands, and 1 outside - lat = latCell(iCell)*180./3.1415 + lat = latCell(iCell)*180.0_RKIND/pi if ( lat>-55.0.and.lat<-50.0 & .or.lat>-35.0.and.lat<-30.0 & .or.lat>-15.0.and.lat<-10.0 & diff --git a/components/mpas-ocean/src/ocean.cmake b/components/mpas-ocean/src/ocean.cmake index 8866a8cea3d6..f2d5303fdd7b 100644 --- a/components/mpas-ocean/src/ocean.cmake +++ b/components/mpas-ocean/src/ocean.cmake @@ -114,6 +114,7 @@ list(APPEND RAW_SOURCES core_ocean/shared/mpas_ocn_stokes_drift.F core_ocean/shared/mpas_ocn_manufactured_solution.F core_ocean/shared/mpas_ocn_subgrid.F + core_ocean/shared/mpas_ocn_scaled_dismf.F ) set(OCEAN_DRIVER diff --git a/components/mpas-ocean/src/shared/Makefile b/components/mpas-ocean/src/shared/Makefile index 35017551c754..27c3db10fee1 100644 --- a/components/mpas-ocean/src/shared/Makefile +++ b/components/mpas-ocean/src/shared/Makefile @@ -143,7 +143,7 @@ mpas_ocn_tracer_advection_std.o: mpas_ocn_config.o mpas_ocn_mesh.o mpas_ocn_trac mpas_ocn_tracer_advection_vert.o: mpas_ocn_mesh.o mpas_ocn_config.o -mpas_ocn_tracer_advection_shared.o: mpas_ocn_mesh.o mpas_ocn_config.o +mpas_ocn_tracer_advection_shared.o: mpas_ocn_constants.o mpas_ocn_mesh.o mpas_ocn_config.o mpas_ocn_tracer_hmix_redi.o: mpas_ocn_constants.o mpas_ocn_config.o @@ -157,7 +157,7 @@ mpas_ocn_tracer_short_wave_absorption_variable.o: mpas_ocn_constants.o mpas_ocn_ mpas_ocn_tracer_short_wave_absorption_jerlov.o: mpas_ocn_constants.o mpas_ocn_config.o -mpas_ocn_vmix.o: mpas_ocn_vmix_cvmix.o mpas_ocn_vmix_coefs_redi.o mpas_ocn_constants.o mpas_ocn_config.o mpas_ocn_diagnostics_variables.o mpas_ocn_vmix_gotm.o +mpas_ocn_vmix.o: mpas_ocn_vmix_cvmix.o mpas_ocn_vmix_coefs_redi.o mpas_ocn_constants.o mpas_ocn_config.o mpas_ocn_diagnostics_variables.o mpas_ocn_vmix_gotm.o mpas_ocn_diagnostics.o mpas_ocn_vmix_cvmix.o: mpas_ocn_constants.o mpas_ocn_config.o mpas_ocn_diagnostics_variables.o mpas_ocn_mesh.o mpas_ocn_stokes_drift.o diff --git a/components/mpas-ocean/src/shared/mpas_ocn_constants.F b/components/mpas-ocean/src/shared/mpas_ocn_constants.F index 35ada1306f8e..dd3214d8d4fc 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_constants.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_constants.F @@ -65,6 +65,7 @@ module ocn_constants real (kind=RKIND), public :: & T0_Kelvin ,&! zero point for Celsius + pi ,&! pi mpercm ,&! meters per m cmperm ,&! m per meter days_per_second ,&! days per second @@ -117,6 +118,7 @@ subroutine ocn_constants_init(configPool, packagePool)!{{{ !----------------------------------------------------------------------- T0_Kelvin = 273.16_RKIND ! zero point for Celsius + pi = 3.14159265358979323846_RKIND ! pi rho_air = 1.2_RKIND ! ambient air density (kg/m^3) rho_sw = config_density0 ! density of salt water (kg/m^3) rho_fw = 1.0e3_RKIND ! avg. water density (kg/m^3) @@ -159,6 +161,7 @@ subroutine ocn_constants_init(configPool, packagePool)!{{{ #ifdef MPAS_ESM_SHR_CONST T0_Kelvin = SHR_CONST_TKFRZ ! zero point for Celsius + pi = SHR_CONST_PI ! zero point for Celsius cp_sw = SHR_CONST_CPSW ! erg/g/K cp_air = SHR_CONST_CPDAIR ! J/kg/K rho_air = SHR_CONST_RHODAIR ! kg/m^3 diff --git a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F index a2d6586a7d63..691bfe70b589 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F @@ -64,7 +64,7 @@ module ocn_diagnostics ocn_fuperp, & ocn_filter_btr_mode_tend_vel, & ocn_reconstruct_eddy_vectors, & - ocn_compute_kpp_input_fields, & + ocn_compute_mixing_input_fields, & ocn_validate_state, & ocn_build_log_filename, & ocn_diagnostics_init @@ -3319,12 +3319,12 @@ end subroutine ocn_filter_btr_mode_tend_vel!}}} !*********************************************************************** ! -! routine ocn_compute_KPP_input_fields +! routine ocn_compute_mixing_input_fields ! !> \brief -!> Compute fields necessary to drive the CVMix KPP module -!> \author Todd Ringler -!> \date 20 August 2013 +!> Compute fields necessary to drive the CVMix KPP and gotm modules +!> \author Todd Ringler, Luke Van Roekel +!> \date 11 July 2024 !> \details !> CVMix/KPP requires the following fields as intent(in): !> surfaceBuoyancyForcing @@ -3333,7 +3333,7 @@ end subroutine ocn_filter_btr_mode_tend_vel!}}} ! !----------------------------------------------------------------------- - subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & + subroutine ocn_compute_mixing_input_fields(statePool, forcingPool, & meshPool, timeLevelIn)!{{{ !----------------------------------------------------------------- @@ -3385,8 +3385,7 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & evapTemperatureFlux, & icebergTemperatureFlux, & seaIceTemperatureFlux, & - surfaceStress, & - surfaceStressMagnitude + sfcStressMag real (kind=RKIND), dimension(:,:), pointer :: & layerThickness, &! layer thickness @@ -3419,7 +3418,7 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & !----------------------------------------------------------------- ! Begin code - call mpas_timer_start('KPP input fields') + call mpas_timer_start('Mixing input fields') if (present(timeLevelIn)) then timeLevel = timeLevelIn @@ -3469,18 +3468,8 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & surfaceThicknessFluxSubglacialRunoff) call mpas_pool_get_array(forcingPool, 'penetrativeTemperatureFlux', & penetrativeTemperatureFlux) - call mpas_pool_get_array(forcingPool, 'surfaceStress', & - surfaceStress) call mpas_pool_get_array(forcingPool, 'surfaceStressMagnitude', & - surfaceStressMagnitude) - call mpas_pool_get_array(forcingPool, 'rainTemperatureFlux', & - rainTemperatureFlux) - call mpas_pool_get_array(forcingPool, 'evapTemperatureFlux', & - evapTemperatureFlux) - call mpas_pool_get_array(forcingPool, 'seaIceTemperatureFlux', & - seaIceTemperatureFlux) - call mpas_pool_get_array(forcingPool, 'icebergTemperatureFlux', & - icebergTemperatureFlux) + sfcStressMag) ! allocate scratch space displaced density computation ncells = nCellsAll @@ -3540,6 +3529,7 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & !$omp parallel !$omp do schedule(runtime) & !$omp private(kmin, fracAbsorbed, fracAbsorbedRunoff, fracAbsorbedSubglacialRunoff, & + !$omp zTop, k, zBot, transmissionCoeffTop, transmissionCoeffBot, & !$omp sumSurfaceStressSquared, i, iEdge) do iCell = 1, nCells @@ -3573,8 +3563,8 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & do k = maxLevelCell(iCell), minLevelCell(iCell), -1 zBot = zTop - layerThickness(k,iCell) if (k == minLevelCell(iCell)) then - transmissionCoeffTop = exp( max(zTop / config_flux_attenuation_coefficient_runoff, -100.0_RKIND) ) - transmissionCoeffBot = exp( max(zBot / config_flux_attenuation_coefficient_runoff, -100.0_RKIND) ) + transmissionCoeffTop = exp( max(zTop / config_flux_attenuation_coefficient_subglacial_runoff, -100.0_RKIND) ) + transmissionCoeffBot = exp( max(zBot / config_flux_attenuation_coefficient_subglacial_runoff, -100.0_RKIND) ) fracAbsorbedSubglacialRunoff = transmissionCoeffTop - transmissionCoeffBot end if zTop = zBot @@ -3593,17 +3583,8 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & activeTracersSurfaceFlux(indexTempFlux,iCell) & + penetrativeTemperatureFlux(iCell) & - penetrativeTemperatureFluxOBL(iCell) & - - fracAbsorbed* (rainTemperatureFlux(iCell) + & - evapTemperatureFlux(iCell) + & - seaIceTemperatureFlux(iCell) + & - icebergTemperatureFlux(iCell)) & - - fracAbsorbedRunoff* & - activeTracersSurfaceFluxRunoff(indexTempFlux,iCell) - if (trim(config_subglacial_runoff_mode) == 'data') then - nonLocalSurfaceTracerFlux(indexTempFlux, iCell) = nonLocalSurfaceTracerFlux(indexTempFlux, iCell) & - - fracAbsorbedSubglacialRunoff* & - activeTracersSurfaceFluxSubglacialRunoff(indexTempFlux,iCell) - end if + - fracAbsorbed*surfaceThicknessFlux(iCell) * & + activeTracers(indexTempFlux,kmin,iCell) nonLocalSurfaceTracerFlux(indexSaltFlux,iCell) = & activeTracersSurfaceFlux(indexSaltFlux,iCell) & @@ -3629,21 +3610,8 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & surfaceBuoyancyForcing(iCell) = surfaceBuoyancyForcing(iCell)* & gravity - ! compute magnitude of surface stress - sumSurfaceStressSquared = 0.0_RKIND - do i = 1, nEdgesOnCell(iCell) - iEdge = edgesOnCell(i, iCell) - sumSurfaceStressSquared = sumSurfaceStressSquared & - + edgeAreaFractionOfCell(i,iCell)* & - surfaceStress(iEdge)**2 - enddo - - ! NOTE that the factor of 2 is from averaging dot products - ! to cell centers on a C-grid - surfaceStressMagnitude(iCell) = & - sqrt(2.0_RKIND * sumSurfaceStressSquared) surfaceFrictionVelocity(iCell) = & - sqrt(surfaceStressMagnitude(iCell) / rho_sw) + sqrt(sfcStressMag(iCell) / rho_sw) enddo !$omp end do @@ -3654,11 +3622,11 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & salineContractionCoeff, & densitySurfaceDisplaced) - call mpas_timer_stop('KPP input fields') + call mpas_timer_stop('Mixing input fields') !------------------------------------------------------------------- - end subroutine ocn_compute_KPP_input_fields!}}} + end subroutine ocn_compute_mixing_input_fields!}}} !*********************************************************************** ! diff --git a/components/mpas-ocean/src/shared/mpas_ocn_gm.F b/components/mpas-ocean/src/shared/mpas_ocn_gm.F index 6f4442103fa7..3db04c4d3e8f 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_gm.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_gm.F @@ -484,7 +484,7 @@ subroutine ocn_GM_compute_Bolus_velocity(statePool, & ! Compute the speed of the first baroclinic mode from the Brunt-Vaisala frequency. cGMphaseSpeed(iEdge) = max(c_min, & - sum_hN/(config_GM_spatially_variable_baroclinic_mode*3.141592_RKIND)) + sum_hN/(config_GM_spatially_variable_baroclinic_mode*pi)) end do !$omp end do @@ -615,7 +615,7 @@ subroutine ocn_GM_compute_Bolus_velocity(statePool, & ! for the first-mode (m=1) baroclinic gravity wave speed in m/s ! c_m = 1/pi integral(N dz ) - c_mode1 = max(1.0E-5_RKIND, sum_hN/3.141592_RKIND) + c_mode1 = max(1.0E-5_RKIND, sum_hN/pi) ! See Hallberg (2013) https://doi.org/10.1016/j.ocemod.2013.08.007 ! just after eqn 4 the defines the deformation diff --git a/components/mpas-ocean/src/shared/mpas_ocn_manufactured_solution.F b/components/mpas-ocean/src/shared/mpas_ocn_manufactured_solution.F index e9ad6786103c..6dd1e1d6e317 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_manufactured_solution.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_manufactured_solution.F @@ -58,6 +58,7 @@ module ocn_manufactured_solution real (kind=RKIND) :: kx, ky real (kind=RKIND) :: ang_freq real (kind=RKIND) :: eta0 + real (kind=RKIND) :: viscDel2, viscDel4 real (kind=RKIND) :: H0 !*********************************************************************** @@ -76,7 +77,13 @@ module ocn_manufactured_solution ! !----------------------------------------------------------------------- - subroutine ocn_manufactured_solution_tend_thick(tend, err)!{{{ + subroutine ocn_manufactured_solution_tend_thick(forcingPool, tend, err)!{{{ + + !----------------------------------------------------------------- + ! input variables + !----------------------------------------------------------------- + + type (mpas_pool_type), intent(in) :: forcingPool !----------------------------------------------------------------- ! input/output variables @@ -97,6 +104,7 @@ subroutine ocn_manufactured_solution_tend_thick(tend, err)!{{{ integer :: iCell, kmin, kmax, k real (kind=RKIND) :: phase, time + real (kind=RKIND), pointer :: forcingTimeIncrement ! End preamble !------------- @@ -104,7 +112,9 @@ subroutine ocn_manufactured_solution_tend_thick(tend, err)!{{{ if (.not. config_use_manufactured_solution) return - time = daysSinceStartOfSim*86400.0_RKIND + call mpas_pool_get_array(forcingPool, 'forcingTimeIncrement', forcingTimeIncrement) + + time = daysSinceStartOfSim*86400.0_RKIND + forcingTimeIncrement do iCell = 1,nCellsOwned @@ -137,7 +147,13 @@ end subroutine ocn_manufactured_solution_tend_thick!}}} !> This routine computes the velocity tendency for the manufactured solution ! !----------------------------------------------------------------------- - subroutine ocn_manufactured_solution_tend_vel(tend, err)!{{{ + subroutine ocn_manufactured_solution_tend_vel(forcingPool, tend, err)!{{{ + + !----------------------------------------------------------------- + ! input variables + !----------------------------------------------------------------- + + type (mpas_pool_type), intent(in) :: forcingPool !----------------------------------------------------------------- ! input/output variables @@ -158,6 +174,7 @@ subroutine ocn_manufactured_solution_tend_vel(tend, err)!{{{ integer :: iEdge, kmin, kmax, k real (kind=RKIND) :: phase, u, v, time + real (kind=RKIND), pointer :: forcingTimeIncrement ! End preamble !----------------------------------------------------------------- @@ -165,7 +182,9 @@ subroutine ocn_manufactured_solution_tend_vel(tend, err)!{{{ if (.not. config_use_manufactured_solution) return - time = daysSinceStartOfSim*86400.0_RKIND + call mpas_pool_get_array(forcingPool, 'forcingTimeIncrement', forcingTimeIncrement) + + time = daysSinceStartOfSim*86400.0_RKIND + forcingTimeIncrement do iEdge = 1, nEdgesOwned @@ -183,6 +202,17 @@ subroutine ocn_manufactured_solution_tend_vel(tend, err)!{{{ + ang_freq*sin(phase) & - 0.5_RKIND*eta0*(kx + ky)*sin(2.0_RKIND*(phase))) + if (.not. config_disable_vel_hmix) then + if (config_use_mom_del2) then + u = u + viscDel2 * eta0 * (kx**2 + ky**2) * cos(phase) + v = v + viscDel2 * eta0 * (kx**2 + ky**2) * cos(phase) + endif + if (config_use_mom_del4) then + u = u - viscDel4 * eta0 * ((kx**4 + ky**4 + kx**2 * ky**2) * cos(phase)) + v = v - viscDel4 * eta0 * ((kx**4 + ky**4 + kx**2 * ky**2) * cos(phase)) + endif + endif + tend(k,iEdge) = tend(k,iEdge) + u*cos(angleEdge(iEdge)) + v*sin(angleEdge(iEdge)) enddo @@ -217,8 +247,11 @@ subroutine ocn_manufactured_solution_init(domain, err)!{{{ if (.not. config_use_manufactured_solution) return - kx = 2.0_RKIND*pii / config_manufactured_solution_wavelength_x - ky = 2.0_RKIND*pii / config_manufactured_solution_wavelength_y + viscDel2 = config_mom_del2 + viscDel4 = config_mom_del4 + + kx = 2.0_RKIND*pi / config_manufactured_solution_wavelength_x + ky = 2.0_RKIND*pi / config_manufactured_solution_wavelength_y eta0 = config_manufactured_solution_amplitude diff --git a/components/mpas-ocean/src/shared/mpas_ocn_mesh.F b/components/mpas-ocean/src/shared/mpas_ocn_mesh.F index 5e0690704e40..9eadb339b660 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_mesh.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_mesh.F @@ -33,6 +33,7 @@ module ocn_mesh use mpas_log use ocn_config + use ocn_constants implicit none private @@ -1792,7 +1793,7 @@ subroutine ocn_meshScaling() !{{{ ! neighboring cells are circles for this calculation. cellWidth = 2.0_RKIND* & sqrt((areaCell(cell1) + areaCell(cell2))/ & - 2.0_RKIND/pii) + 2.0_RKIND/pi) meshScalingDel2(iEdge) = cellWidth/ & config_hmix_ref_cell_width meshScalingDel4(iEdge) = (cellWidth/ & diff --git a/components/mpas-ocean/src/shared/mpas_ocn_scaled_dismf.F b/components/mpas-ocean/src/shared/mpas_ocn_scaled_dismf.F new file mode 100644 index 000000000000..bfdb39e49ad6 --- /dev/null +++ b/components/mpas-ocean/src/shared/mpas_ocn_scaled_dismf.F @@ -0,0 +1,309 @@ +! Copyright (c) 2013, Los Alamos National Security, LLC (LANS) +! and the University Corporation for Atmospheric Research (UCAR). +! +! Unless noted otherwise source code is licensed under the BSD license. +! Additional copyright and license information can be found in the LICENSE file +! distributed with this code, or at http://mpas-dev.github.io/license.html +! +!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +! +! ocn_scaled_dismf +!> \brief MPAS ocean scale data ice-shelf melt fluxes +!> \author Xylar Asay-Davis +!> \date July 2024 +!> \details +!> This module contains routines for scaling data ice-shelf melt fluxes by +!> the running mean of the removed ice runoff +!> Design document located at: +!> https://acme-climate.atlassian.net/wiki/spaces/PSC/pages/4210098268/Design+Document+Data+iceberg+and+ice-shelf+melt+flux+patterns+for+E3SM+runs +! +!----------------------------------------------------------------------- + +module ocn_scaled_dismf + + use mpas_kind_types + use mpas_derived_types + use mpas_global_sum_mod + use mpas_timekeeping + use mpas_timer + + use ocn_config + use ocn_mesh + + use shr_const_mod + + implicit none + private + save + + !-------------------------------------------------------------------- + ! + ! Public parameters + ! + !-------------------------------------------------------------------- + + !-------------------------------------------------------------------- + ! + ! Public member functions + ! + !-------------------------------------------------------------------- + + public :: ocn_init_scaled_dismf, & + ocn_update_scaled_dismf + + !-------------------------------------------------------------------- + ! + ! Private module variables + ! + !-------------------------------------------------------------------- + + logical :: scaledDISMFOn + character (len=*), parameter :: alarmID = 'scaledDISMFUpdateAlarm' + +!*********************************************************************** + +contains + + +!*********************************************************************** +! +! routine ocn_init_scaled_dismf +! +!> \brief Initialize scaling of data ice-shelf melt fluxes +!> \author Xylar Asay-Davis +!> \date July 2024 +!> \details +!> Set alarms needed to compute daily and running means of removed ice runoff +! +!----------------------------------------------------------------------- + + subroutine ocn_init_scaled_dismf(domain)!{{{ + + type (domain_type), intent(inout) :: domain + + ! Alarm variables + type (MPAS_Time_Type) :: alarmTime + type (MPAS_TimeInterval_type) :: alarmTimeStep + + integer :: err_tmp + + if (config_scale_dismf_by_removed_ice_runoff) then + scaledDISMFOn = .true. + else + scaledDISMFOn = .false. + return + endif + + if (.not. config_remove_ais_ice_runoff) then + call mpas_log_write('config_scale_dismf_by_removed_ice_runoff = .true. requires config_remove_ais_ice_runoff = .true.', & + MPAS_LOG_CRIT) + endif + + if (trim(config_land_ice_flux_mode) /= 'data') then + call mpas_log_write('config_scale_dismf_by_removed_ice_runoff = .true. requires config_land_ice_flux_mode = "data"', & + MPAS_LOG_CRIT) + endif + + ! Setup Alarm for updating of scaled DISMF + alarmTime = mpas_get_clock_time(domain % clock, & + MPAS_START_TIME, & + ierr=err_tmp) + call mpas_set_timeInterval(alarmTimeStep, & + timeString='0000-00-01_00:00:00', & + ierr=err_tmp) + call mpas_add_clock_alarm(domain % clock, alarmID, alarmTime + alarmTimeStep, & + alarmTimeInterval=alarmTimeStep, ierr=err_tmp) + + end subroutine ocn_init_scaled_dismf!}}} + + +!*********************************************************************** +! +! routine ocn_update_scaled_dismf +! +!> \brief Update scaled data ice-shelf melt fluxes +!> \author Xylar Asay-Davis +!> \date August 2024 +!> \details +!> Accumulate daily mean of removed runoff. If we are at the end of a day, +!> update the running mean of removed ice runoff, and use it to scale +!> the ice shelf melt flux based on the pattern from a data file +! +!----------------------------------------------------------------------- + + subroutine ocn_update_scaled_dismf(domain)!{{{ + + type (domain_type), intent(inout) :: domain + + integer :: err_tmp + + if (.not. scaledDISMFOn) then + return + end if + + ! since the current clock time is for the end of the accumulation + ! intereval, first, accumulate the daily mean + call accumulate_mean_removed_ice_runoff(domain) + + ! then, compute update the history and running mean if we are at the + ! end of a day + if(mpas_is_alarm_ringing(domain % clock, alarmID, ierr=err_tmp)) then +#ifdef MPAS_DEBUG + call mpas_log_write(' Computing Scaled DISMF') +#endif + call update_scaled_dismf(domain) + call mpas_reset_clock_alarm(domain % clock, alarmID, ierr=err_tmp) + endif + + end subroutine ocn_update_scaled_dismf!}}} + + +!*********************************************************************** +! +! routine accumulate_mean_removed_ice_runoff +! +!> \brief Accumulate mean removed ice runoff +!> \author Xylar Asay-Davis +!> \date August 2024 +!> \details +!> Accumulate current removed ice runoff into the daily mean value +! +!----------------------------------------------------------------------- + + subroutine accumulate_mean_removed_ice_runoff(domain)!{{{ + + type (domain_type), intent(inout) :: domain + + type (block_type), pointer :: block_ptr + type (mpas_pool_type), pointer :: forcingPool + + real (kind=RKIND), dimension(:), pointer :: removedIceRunoffFlux + real (kind=RKIND), pointer :: avgRemovedIceRunoff + integer, pointer :: nCellsSolve, nAccumulated + + real (kind=RKIND) :: totalFlux + real (kind=RKIND), dimension(:), allocatable :: localArrayForReproSum + + integer :: iCell + + block_ptr => domain % blocklist + + call mpas_pool_get_subpool(block_ptr % structs, 'forcing', forcingPool) + + ! independent of blocks + call mpas_pool_get_array(forcingPool, 'avgRemovedIceRunoff', avgRemovedIceRunoff) + call mpas_pool_get_dimension(forcingPool, 'nCellsSolve', nCellsSolve) + call mpas_pool_get_array(forcingPool, 'nAccumulatedRemovedIceRunoff', nAccumulated) + + do while(associated(block_ptr)) + call mpas_pool_get_subpool(block_ptr % structs, 'forcing', forcingPool) + + call mpas_pool_get_array(forcingPool, 'removedIceRunoffFlux', removedIceRunoffFlux) + call mpas_pool_get_array(forcingPool, 'avgRemovedIceRunoff', avgRemovedIceRunoff) + + call mpas_pool_get_dimension(forcingPool, 'nCellsSolve', nCellsSolve) + + call mpas_pool_get_array(forcingPool, 'nAccumulatedRemovedIceRunoff', nAccumulated) + + + allocate(localArrayForReproSum(nCellsSolve)) + localArrayForReproSum(:) = 0.0_RKIND + + !$omp parallel + !$omp do schedule(runtime) + do iCell=1,nCellsSolve + localArrayForReproSum(iCell) = removedIceRunoffFlux(iCell) * areaCell(iCell) + enddo + !$omp end do + !$omp end parallel + + block_ptr => block_ptr % next + end do ! block_ptr + + totalFlux = mpas_global_sum(localArrayForReproSum, domain % dminfo % comm) + + deallocate (localArrayForReproSum) + + avgRemovedIceRunoff = (avgRemovedIceRunoff * nAccumulated + totalFlux) & + / ( nAccumulated + 1 ) + + nAccumulated = nAccumulated + 1 + + end subroutine accumulate_mean_removed_ice_runoff!}}} + + + +!*********************************************************************** +! +! routine update_scaled_dismf +! +!> \brief Update scaled data ice-shelf melt fluxes +!> \author Xylar Asay-Davis +!> \date August 2024 +!> \details +!> Update the running mean of removed ice runoff, and use it to scale +!> the ice shelf melt flux based on the pattern from a data file +! +!----------------------------------------------------------------------- + + subroutine update_scaled_dismf(domain)!{{{ + + type (domain_type), intent(inout) :: domain + + type (block_type), pointer :: block_ptr + type (mpas_pool_type), pointer :: forcingPool + + real (kind=RKIND), pointer :: avgRemovedIceRunoff, runningMeanRemovedIceRunoff + real (kind=RKIND), dimension(:), pointer :: totalRemovedIceRunoffHistory + integer, pointer :: nValidHistory, nAccumulated + + real (kind=RKIND), dimension(:), allocatable :: tmpHistory + + real (kind=RKIND) :: previousTotal, timeInterval + integer :: nHistory + + block_ptr => domain % blocklist + call mpas_pool_get_subpool(block_ptr % structs, 'forcing', forcingPool) + + call mpas_pool_get_array(forcingPool, 'nAccumulatedRemovedIceRunoff', nAccumulated) + call mpas_pool_get_array(forcingPool, 'nValidTotalRemovedIceRunoffHistory', nValidHistory) + call mpas_pool_get_array(forcingPool, 'avgRemovedIceRunoff', avgRemovedIceRunoff) + call mpas_pool_get_array(forcingPool, 'totalRemovedIceRunoffHistory', totalRemovedIceRunoffHistory) + call mpas_pool_get_array(forcingPool, 'runningMeanRemovedIceRunoff', runningMeanRemovedIceRunoff) + + nHistory = config_ais_ice_runoff_history_days + + previousTotal = totalRemovedIceRunoffHistory(nValidHistory) + + if (nValidHistory == 0) then + ! we keep zero as the first entry in totalRemovedIceRunoffHistory + nValidHistory = 1 + end if + + if (nValidHistory == nHistory) then + ! we need to shift the history, since it's full + allocate(tmpHistory(nHistory)) + tmpHistory(:) = totalRemovedIceRunoffHistory(:) + totalRemovedIceRunoffHistory(1:nHistory - 1) = tmpHistory(2:nHistory) + else + ! the history isn't full yet, so we just add to the end + nValidHistory = nValidHistory + 1 + end if + + ! add the new total, the previous total plus the new daily average + totalRemovedIceRunoffHistory(nValidHistory) = previousTotal + SHR_CONST_CDAY * avgRemovedIceRunoff + + timeInterval = SHR_CONST_CDAY * (nValidHistory - 1) + ! the running mean is the difference between the newest and oldest + ! totals divided by the time between them + runningMeanRemovedIceRunoff = & + (totalRemovedIceRunoffHistory(nValidHistory) - totalRemovedIceRunoffHistory(1)) & + / timeInterval + + ! reset daily averaging of the removed runoff + nAccumulated = 0 + avgRemovedIceRunoff = 0.0_RKIND + + end subroutine update_scaled_dismf!}}} + +end module ocn_scaled_dismf \ No newline at end of file diff --git a/components/mpas-ocean/src/shared/mpas_ocn_surface_land_ice_fluxes.F b/components/mpas-ocean/src/shared/mpas_ocn_surface_land_ice_fluxes.F index c249f202757d..db5df3dfbe00 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_surface_land_ice_fluxes.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_surface_land_ice_fluxes.F @@ -278,6 +278,12 @@ subroutine ocn_surface_land_ice_fluxes_thick(forcingPool, surfaceThicknessFlux, real (kind=RKIND), dimension(:), pointer :: landIceFreshwaterFlux, & dataLandIceFreshwaterFlux + real (kind=RKIND), pointer :: runningMeanRemovedIceRunoff, & + areaIntegAnnMeanDataIcebergIceShelfFreshwaterFlux + + + real (kind=RKIND) :: scaling + err = 0 if (.not.landIceFluxesOn) return @@ -287,6 +293,16 @@ subroutine ocn_surface_land_ice_fluxes_thick(forcingPool, surfaceThicknessFlux, call mpas_pool_get_array(forcingPool, 'landIceFreshwaterFlux', landIceFreshwaterFlux) if ( landIceDataOn ) then + + if (config_scale_dismf_by_removed_ice_runoff) then + call mpas_pool_get_array(forcingPool, 'runningMeanRemovedIceRunoff', runningMeanRemovedIceRunoff) + call mpas_pool_get_array(forcingPool, 'areaIntegAnnMeanDataIcebergIceShelfFreshwaterFlux', & + areaIntegAnnMeanDataIcebergIceShelfFreshwaterFlux) + scaling = runningMeanRemovedIceRunoff / areaIntegAnnMeanDataIcebergIceShelfFreshwaterFlux + else + scaling = 1.0_RKIND + end if + call mpas_pool_get_array(forcingPool, 'dataLandIceFreshwaterFlux', dataLandIceFreshwaterFlux) #ifdef MPAS_OPENACC !$acc enter data copyin(landIceFreshwaterFlux) @@ -297,7 +313,7 @@ subroutine ocn_surface_land_ice_fluxes_thick(forcingPool, surfaceThicknessFlux, !$omp do schedule(runtime) #endif do iCell = 1, nCellsAll - landIceFreshwaterFlux(iCell) = dataLandIceFreshwaterFlux(iCell) + landIceFreshwaterFlux(iCell) = scaling * dataLandIceFreshwaterFlux(iCell) end do #ifndef MPAS_OPENACC !$omp end do @@ -378,6 +394,12 @@ subroutine ocn_surface_land_ice_fluxes_active_tracers(meshPool, forcingPool, tra real (kind=RKIND), dimension(:), pointer :: landIceHeatFlux, & dataLandIceHeatFlux + real (kind=RKIND), pointer :: runningMeanRemovedIceRunoff, & + areaIntegAnnMeanDataIcebergIceShelfFreshwaterFlux + + + real (kind=RKIND) :: scaling + err = 0 if (.not.landIceFluxesOn) return @@ -387,11 +409,21 @@ subroutine ocn_surface_land_ice_fluxes_active_tracers(meshPool, forcingPool, tra call mpas_pool_get_array(forcingPool, 'landIceHeatFlux', landIceHeatFlux) if ( landIceDataOn ) then + + if (config_scale_dismf_by_removed_ice_runoff) then + call mpas_pool_get_array(forcingPool, 'runningMeanRemovedIceRunoff', runningMeanRemovedIceRunoff) + call mpas_pool_get_array(forcingPool, 'areaIntegAnnMeanDataIcebergIceShelfFreshwaterFlux', & + areaIntegAnnMeanDataIcebergIceShelfFreshwaterFlux) + scaling = runningMeanRemovedIceRunoff / areaIntegAnnMeanDataIcebergIceShelfFreshwaterFlux + else + scaling = 1.0_RKIND + end if + call mpas_pool_get_array(forcingPool, 'dataLandIceHeatFlux', dataLandIceHeatFlux) !$omp parallel !$omp do schedule(runtime) do iCell = 1, nCells - landIceHeatFlux(iCell) = dataLandIceHeatFlux(iCell) + landIceHeatFlux(iCell) = scaling * dataLandIceHeatFlux(iCell) end do !$omp end do !$omp end parallel @@ -492,168 +524,179 @@ subroutine ocn_surface_land_ice_fluxes_build_arrays(meshPool, & err = 0 - if (.not.landIceStandaloneOn) return + if (.not. (landIceStandaloneOn .or. landIceDataOn)) return call mpas_timer_start("land_ice_build_arrays") call mpas_pool_get_dimension(meshPool, 'nCellsArray', nCellsArray) - call mpas_pool_get_array(forcingPool, 'landIcePressure', landIcePressure) - - call mpas_pool_get_array(forcingPool, 'landIceFloatingFraction', landIceFloatingFraction) - call mpas_pool_get_array(forcingPool, 'landIceFloatingMask', landIceFloatingMask) - call mpas_pool_get_array(forcingPool, 'landIceFreshwaterFlux', landIceFreshwaterFlux) - call mpas_pool_get_array(forcingPool, 'landIceHeatFlux', landIceHeatFlux) - call mpas_pool_get_array(forcingPool, 'heatFluxToLandIce', heatFluxToLandIce) - call mpas_pool_get_array(forcingPool, 'frazilIceFreshwaterFlux', frazilIceFreshwaterFlux) call mpas_pool_get_array(forcingPool, 'landIceFreshwaterFluxTotal', landIceFreshwaterFluxTotal) - - call mpas_pool_get_array(forcingPool, 'landIceInterfaceTracers', landIceInterfaceTracers) - call mpas_pool_get_dimension(forcingPool, & - 'index_landIceInterfaceTemperature', & - indexITPtr) - call mpas_pool_get_dimension(forcingPool, & - 'index_landIceInterfaceSalinity', & - indexISPtr) - indexIT = indexITPtr - indexIS = indexISPtr - - if (useHollandJenkinsAdvDiff) then - call mpas_pool_get_array(forcingPool, 'landIceSurfaceTemperature', landIceSurfaceTemperature) - - allocate(freezeInterfaceSalinity(nCells), & - freezeInterfaceTemperature(nCells), & - freezeFreshwaterFlux(nCells), & - freezeHeatFlux(nCells), & - freezeIceHeatFlux(nCells)) - end if + call mpas_pool_get_array(forcingPool, 'landIceFloatingMask', landIceFloatingMask) nCells = nCellsArray( size(nCellsArray) ) - if (isomipOn) then !*** ISOMIP formulation + if (landIceStandaloneOn) then - !$omp parallel - !$omp do schedule(runtime) private(freshwaterFlux, heatFlux) - do iCell = 1, nCells - if (landIceFloatingMask(iCell) == 0) cycle + call mpas_pool_get_array(forcingPool, 'landIcePressure', landIcePressure) + call mpas_pool_get_array(forcingPool, 'landIceFloatingFraction', landIceFloatingFraction) + call mpas_pool_get_array(forcingPool, 'landIceHeatFlux', landIceHeatFlux) + call mpas_pool_get_array(forcingPool, 'heatFluxToLandIce', heatFluxToLandIce) + + + call mpas_pool_get_array(forcingPool, 'landIceInterfaceTracers', landIceInterfaceTracers) + call mpas_pool_get_dimension(forcingPool, & + 'index_landIceInterfaceTemperature', & + indexITPtr) + call mpas_pool_get_dimension(forcingPool, & + 'index_landIceInterfaceSalinity', & + indexISPtr) + indexIT = indexITPtr + indexIS = indexISPtr + + if (useHollandJenkinsAdvDiff) then + call mpas_pool_get_array(forcingPool, 'landIceSurfaceTemperature', landIceSurfaceTemperature) + + allocate(freezeInterfaceSalinity(nCells), & + freezeInterfaceTemperature(nCells), & + freezeFreshwaterFlux(nCells), & + freezeHeatFlux(nCells), & + freezeIceHeatFlux(nCells)) + end if - ! linearized equaiton for the S and p dependent potential freezing temperature - landIceInterfaceTracers(indexIT,iCell) = ocn_freezing_temperature( & - salinity=landIceBoundaryLayerTracers(indexBLT,iCell), & - pressure=landIcePressure(iCell), & - inLandIceCavity=.true.) + if (isomipOn) then !*** ISOMIP formulation - ! using (3) and (4) from Hunter (2006) - ! or (7) from Jenkins et al. (2001) if gamma constant - ! and no heat flux into ice - ! freshwater flux = density * melt rate is in kg/m^2/s - freshwaterFlux = -rho_sw * ISOMIPgammaT * (cp_sw/latent_heat_fusion_mks) & - * (landIceInterfaceTracers(indexIT,iCell)-landIceBoundaryLayerTracers(indexBLT,iCell)) + !$omp parallel + !$omp do schedule(runtime) private(freshwaterFlux, heatFlux) + do iCell = 1, nCells + if (landIceFloatingMask(iCell) == 0) cycle - landIceFreshwaterFlux(iCell) = landIceFloatingFraction(iCell)*freshwaterFlux + ! linearized equaiton for the S and p dependent potential freezing temperature + landIceInterfaceTracers(indexIT,iCell) = ocn_freezing_temperature( & + salinity=landIceBoundaryLayerTracers(indexBLT,iCell), & + pressure=landIcePressure(iCell), & + inLandIceCavity=.true.) - ! Using (13) from Jenkins et al. (2001) - ! heat flux is in W/s - heatFlux = cp_sw*(freshwaterFlux*landIceInterfaceTracers(indexIT,iCell) & - + rho_sw*ISOMIPgammaT & - * (landIceInterfaceTracers(indexIT,iCell)-landIceBoundaryLayerTracers(indexBLT,iCell))) - landIceHeatFlux(iCell) = landIceFloatingFraction(iCell)*heatFlux + ! using (3) and (4) from Hunter (2006) + ! or (7) from Jenkins et al. (2001) if gamma constant + ! and no heat flux into ice + ! freshwater flux = density * melt rate is in kg/m^2/s + freshwaterFlux = -rho_sw * ISOMIPgammaT * (cp_sw/latent_heat_fusion_mks) & + * (landIceInterfaceTracers(indexIT,iCell)-landIceBoundaryLayerTracers(indexBLT,iCell)) - heatFluxToLandIce(iCell) = 0.0_RKIND + landIceFreshwaterFlux(iCell) = landIceFloatingFraction(iCell)*freshwaterFlux - end do - !$omp end do - !$omp end parallel - endif ! isomipOn + ! Using (13) from Jenkins et al. (2001) + ! heat flux is in W/s + heatFlux = cp_sw*(freshwaterFlux*landIceInterfaceTracers(indexIT,iCell) & + + rho_sw*ISOMIPgammaT & + * (landIceInterfaceTracers(indexIT,iCell)-landIceBoundaryLayerTracers(indexBLT,iCell))) + landIceHeatFlux(iCell) = landIceFloatingFraction(iCell)*heatFlux - if (jenkinsOn .or. hollandJenkinsOn) then - if(useHollandJenkinsAdvDiff) then - ! melting solution - call compute_HJ99_melt_fluxes( & - landIceFloatingMask, & - landIceBoundaryLayerTracers(indexBLT,:), & - landIceBoundaryLayerTracers(indexBLS,:), & - landIceTracerTransferVelocities(indexHeatTrans,:), & - landIceTracerTransferVelocities(indexSaltTrans,:), & - landIceSurfaceTemperature, & - landIcePressure, & - landIceInterfaceTracers(indexIT,:), & - landIceInterfaceTracers(indexIS,:), & - landIceFreshwaterFlux, & - landIceHeatFlux, & - heatFluxToLandIce, & - nCells, & - err) - if(err .ne. 0) then - call mpas_log_write( & - 'compute_HJ99_melt_fluxes failed.', & - MPAS_LOG_CRIT) - end if + heatFluxToLandIce(iCell) = 0.0_RKIND - ! freezing solution - call compute_melt_fluxes( & - landIceFloatingMask, & - landIceBoundaryLayerTracers(indexBLT,:), & - landIceBoundaryLayerTracers(indexBLS,:), & - landIceTracerTransferVelocities(indexHeatTrans,:), & - landIceTracerTransferVelocities(indexSaltTrans,:), & - landIcePressure, & - freezeInterfaceTemperature, & - freezeInterfaceSalinity, & - freezeFreshwaterFlux, & - freezeHeatFlux, & - freezeIceHeatFlux, & - nCells, & - err) - if(err .ne. 0) then - call mpas_log_write( & - 'compute_melt_fluxes failed.', & - MPAS_LOG_CRIT) + end do + !$omp end do + !$omp end parallel + end if ! isomipOn + + if (jenkinsOn .or. hollandJenkinsOn) then + if(useHollandJenkinsAdvDiff) then + ! melting solution + call compute_HJ99_melt_fluxes( & + landIceFloatingMask, & + landIceBoundaryLayerTracers(indexBLT,:), & + landIceBoundaryLayerTracers(indexBLS,:), & + landIceTracerTransferVelocities(indexHeatTrans,:), & + landIceTracerTransferVelocities(indexSaltTrans,:), & + landIceSurfaceTemperature, & + landIcePressure, & + landIceInterfaceTracers(indexIT,:), & + landIceInterfaceTracers(indexIS,:), & + landIceFreshwaterFlux, & + landIceHeatFlux, & + heatFluxToLandIce, & + nCells, & + err) + if(err .ne. 0) then + call mpas_log_write( & + 'compute_HJ99_melt_fluxes failed.', & + MPAS_LOG_CRIT) + end if + + ! freezing solution + call compute_melt_fluxes( & + landIceFloatingMask, & + landIceBoundaryLayerTracers(indexBLT,:), & + landIceBoundaryLayerTracers(indexBLS,:), & + landIceTracerTransferVelocities(indexHeatTrans,:), & + landIceTracerTransferVelocities(indexSaltTrans,:), & + landIcePressure, & + freezeInterfaceTemperature, & + freezeInterfaceSalinity, & + freezeFreshwaterFlux, & + freezeHeatFlux, & + freezeIceHeatFlux, & + nCells, & + err) + if(err .ne. 0) then + call mpas_log_write( & + 'compute_melt_fluxes failed.', & + MPAS_LOG_CRIT) + end if + + do iCell = 1, nCells + if ((landIceFloatingMask(iCell) == 0) .or. (landIceFreshwaterFlux(iCell) >= 0.0_RKIND)) cycle + + landIceInterfaceTracers(indexIS,iCell) = freezeInterfaceSalinity(iCell) + landIceInterfaceTracers(indexIT,iCell) = freezeInterfaceTemperature(iCell) + landIceFreshwaterFlux(iCell) = freezeFreshwaterFlux(iCell) + landIceHeatFlux(iCell) = freezeHeatFlux(iCell) + heatFluxToLandIce(iCell) = freezeIceHeatFlux(iCell) + end do + else ! not using Holland and Jenkins advection/diffusion + call compute_melt_fluxes( & + landIceFloatingMask, & + landIceBoundaryLayerTracers(indexBLT,:), & + landIceBoundaryLayerTracers(indexBLS,:), & + landIceTracerTransferVelocities(indexHeatTrans,:), & + landIceTracerTransferVelocities(indexSaltTrans,:), & + landIcePressure, & + landIceInterfaceTracers(indexIT,:), & + landIceInterfaceTracers(indexIS,:), & + landIceFreshwaterFlux, & + landIceHeatFlux, & + heatFluxToLandIce, & + nCells, & + err) + if(err .ne. 0) then + call mpas_log_write( & + 'compute_melt_fluxes failed.', & + MPAS_LOG_CRIT) + end if end if + ! modulate the fluxes by the landIceFloatingFraction do iCell = 1, nCells - if ((landIceFloatingMask(iCell) == 0) .or. (landIceFreshwaterFlux(iCell) >= 0.0_RKIND)) cycle + if (landIceFloatingMask(iCell) == 0) cycle - landIceInterfaceTracers(indexIS,iCell) = freezeInterfaceSalinity(iCell) - landIceInterfaceTracers(indexIT,iCell) = freezeInterfaceTemperature(iCell) - landIceFreshwaterFlux(iCell) = freezeFreshwaterFlux(iCell) - landIceHeatFlux(iCell) = freezeHeatFlux(iCell) - heatFluxToLandIce(iCell) = freezeIceHeatFlux(iCell) + landIceFreshwaterFlux(iCell) = landIceFloatingFraction(iCell)*landIceFreshwaterFlux(iCell) + landIceHeatFlux(iCell) = landIceFloatingFraction(iCell)*landIceHeatFlux(iCell) + heatFluxToLandIce(iCell) = landIceFloatingFraction(iCell)*heatFluxToLandIce(iCell) end do - else ! not using Holland and Jenkins advection/diffusion - call compute_melt_fluxes( & - landIceFloatingMask, & - landIceBoundaryLayerTracers(indexBLT,:), & - landIceBoundaryLayerTracers(indexBLS,:), & - landIceTracerTransferVelocities(indexHeatTrans,:), & - landIceTracerTransferVelocities(indexSaltTrans,:), & - landIcePressure, & - landIceInterfaceTracers(indexIT,:), & - landIceInterfaceTracers(indexIS,:), & - landIceFreshwaterFlux, & - landIceHeatFlux, & - heatFluxToLandIce, & - nCells, & - err) - if(err .ne. 0) then - call mpas_log_write( & - 'compute_melt_fluxes failed.', & - MPAS_LOG_CRIT) - end if - end if - ! modulate the fluxes by the landIceFloatingFraction - do iCell = 1, nCells - if (landIceFloatingMask(iCell) == 0) cycle + end if ! jenkinsOn or hollandJenkinsOn - landIceFreshwaterFlux(iCell) = landIceFloatingFraction(iCell)*landIceFreshwaterFlux(iCell) - landIceHeatFlux(iCell) = landIceFloatingFraction(iCell)*landIceHeatFlux(iCell) - heatFluxToLandIce(iCell) = landIceFloatingFraction(iCell)*heatFluxToLandIce(iCell) - end do + if(useHollandJenkinsAdvDiff) then + deallocate(freezeInterfaceSalinity, & + freezeInterfaceTemperature, & + freezeFreshwaterFlux, & + freezeHeatFlux, & + freezeIceHeatFlux) + end if - endif ! jenkinsOn or hollandJenkinsOn + end if ! landIceStandaloneOn ! Add frazil and interface melt/freeze to get total fresh water flux if ( associated(frazilIceFreshwaterFlux) ) then @@ -666,14 +709,6 @@ subroutine ocn_surface_land_ice_fluxes_build_arrays(meshPool, & end do end if - if(useHollandJenkinsAdvDiff) then - deallocate(freezeInterfaceSalinity, & - freezeInterfaceTemperature, & - freezeFreshwaterFlux, & - freezeHeatFlux, & - freezeIceHeatFlux) - end if - call mpas_timer_stop("land_ice_build_arrays") !-------------------------------------------------------------------- diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F index f85427913b09..0d049d1eb422 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F @@ -264,7 +264,8 @@ subroutine ocn_tend_thick(tendPool, forcingPool)!{{{ tendThick, err) ! Compute thickness tendency to manufactured solution - call ocn_manufactured_solution_tend_thick(tendThick, err) + call ocn_manufactured_solution_tend_thick(forcingPool, & + tendThick, err) #ifdef MPAS_OPENACC !$acc exit data copyout(tendThick, surfaceThicknessFlux, surfaceThicknessFluxRunoff, & @@ -493,7 +494,8 @@ subroutine ocn_tend_vel(domain, tendPool, statePool, forcingPool, & layerThickEdgeMean, tendVel, err) ! Compute tendency term for manufactured solution - call ocn_manufactured_solution_tend_vel(tendVel, err) + call ocn_manufactured_solution_tend_vel(forcingPool, & + tendVel, err) ! vertical mixing treated implicitly in a later routine ! adjust total velocity tendency based on wetting/drying @@ -1317,12 +1319,9 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, & ! ! Compute tracer tendency due to non-local flux computed in KPP ! - if (config_use_cvmix_kpp .or. config_use_gotm) then - call ocn_compute_KPP_input_fields(statePool, forcingPool,& - meshPool, timeLevel) - - if (.not. config_cvmix_kpp_nonlocal_with_implicit_mix) then + if (config_use_cvmix_kpp) then call mpas_timer_start("non-local flux from KPP") + if (.not. config_cvmix_kpp_nonlocal_with_implicit_mix) then if (computeBudgets) then !$omp parallel !$omp do schedule(runtime) private(k,n) @@ -1365,8 +1364,8 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, & !$omp end do !$omp end parallel endif ! compute budgets - call mpas_timer_stop("non-local flux from KPP") end if ! not non-local with implicit mix + call mpas_timer_stop("non-local flux from KPP") end if ! KPP ! Compute tracer tendency due to production/destruction of @@ -1527,7 +1526,7 @@ subroutine ocn_tend_freq_filtered_thickness(tendPool, statePool, & do k = 1, maxLevelCell(iCell) tend_lowFreqDivergence(k,iCell) = & - -2.0*pii/thickness_filter_timescale_sec & + -2.0*pi/thickness_filter_timescale_sec & *(lowFreqDivergence(k,iCell) - div_hu(k) & + div_hu_btr * layerThickness(k,iCell)/totalThickness) @@ -1535,7 +1534,7 @@ subroutine ocn_tend_freq_filtered_thickness(tendPool, statePool, & div_hu_btr*layerThickness(k,iCell)/totalThickness + & lowFreqDivergence(k,iCell) + & use_highFreqThick_restore* & - (-2.0*pii/highFreqThick_restore_time_sec* & + (-2.0*pi/highFreqThick_restore_time_sec* & highFreqThickness(k,iCell) ) end do ! vert (k) loop diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tidal_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_tidal_forcing.F index 2a9b400dd90b..48901fbf5d51 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tidal_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tidal_forcing.F @@ -243,8 +243,8 @@ subroutine ocn_tidal_forcing_build_array(domain, meshPool, forcingPool, statePoo ! compute the tidalHeight if (trim(config_tidal_forcing_model) == 'monochromatic') then tidalHeight = config_tidal_forcing_monochromatic_amp * & - SIN(2.0_RKIND*pii/config_tidal_forcing_monochromatic_period * daysSinceStartOfSim - & - pii*config_tidal_forcing_monochromatic_phaseLag/180.0_RKIND) - & + SIN(2.0_RKIND*pi/config_tidal_forcing_monochromatic_period * daysSinceStartOfSim - & + pi*config_tidal_forcing_monochromatic_phaseLag/180.0_RKIND) - & config_tidal_forcing_monochromatic_baseline elseif (trim(config_tidal_forcing_model) == 'linear') then tidalHeight = max(config_tidal_forcing_linear_min, & diff --git a/components/mpas-ocean/src/shared/mpas_ocn_time_average_coupled.F b/components/mpas-ocean/src/shared/mpas_ocn_time_average_coupled.F index c8ce0b8dc151..0b74b7c56b81 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_time_average_coupled.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_time_average_coupled.F @@ -25,6 +25,8 @@ module ocn_time_average_coupled use mpas_pool_routines use ocn_constants use ocn_config + use ocn_equation_of_state + use ocn_mesh use ocn_tracer_ecosys use ocn_tracer_DMS use ocn_tracer_MacroMolecules @@ -58,6 +60,8 @@ subroutine ocn_time_average_coupled_init(forcingPool)!{{{ avgRemovedRiverRunoffFlux, avgRemovedIceRunoffFlux, & avgLandIceHeatFlux, avgRemovedIceRunoffHeatFlux + real (kind=RKIND), dimension(:), pointer :: avgThermalForcingAtCritDepth + integer :: iCell integer, pointer :: nAccumulatedCoupled, nCells @@ -160,6 +164,18 @@ subroutine ocn_time_average_coupled_init(forcingPool)!{{{ !$omp end parallel end if + if (trim(config_glc_thermal_forcing_coupling_mode) == '2d') then + call mpas_pool_get_array(forcingPool, 'avgThermalForcingAtCritDepth', avgThermalForcingAtCritDepth) + + !$omp parallel + !$omp do schedule(runtime) + do iCell = 1, nCells + avgThermalForcingAtCritDepth(iCell) = 0.0_RKIND + end do + !$omp end do + !$omp end parallel + endif + ! set up BGC coupling fields if necessary if (config_use_ecosysTracers) then @@ -257,7 +273,7 @@ subroutine ocn_time_average_coupled_accumulate(statePool, forcingPool, timeLevel real (kind=RKIND), dimension(:,:), pointer :: avgSSHGradient integer :: iCell integer, pointer :: index_temperaturePtr, index_SSHzonalPtr, & - index_SSHmeridionalPtr, nAccumulatedCoupled, nCells + index_SSHmeridionalPtr, nAccumulatedCoupled, nCells, nVertLevels integer :: index_temperature, index_SSHzonal, index_SSHmeridional real (kind=RKIND), dimension(:,:), pointer :: & avgLandIceBoundaryLayerTracers, avgLandIceTracerTransferVelocities @@ -267,10 +283,16 @@ subroutine ocn_time_average_coupled_accumulate(statePool, forcingPool, timeLevel landIceHeatFlux, avgLandIceHeatFlux, & removedRiverRunoffFlux, avgRemovedRiverRunoffFlux, & removedIceRunoffFlux, avgRemovedIceRunoffFlux, & - avgRemovedIceRunoffHeatFlux + avgRemovedIceRunoffHeatFlux, & + avgThermalForcingAtCritDepth type (mpas_pool_type), pointer :: tracersPool + real (kind=RKIND), dimension(:,:,:), pointer :: activeTracers + integer, pointer :: indexTemperature, indexSalinity + integer :: iLevelCritDepth, iLevel + real (kind=RKIND) :: freezingTemp + real (kind=RKIND), dimension(:,:,:), pointer :: & ecosysTracers, & DMSTracers, & @@ -306,6 +328,7 @@ subroutine ocn_time_average_coupled_accumulate(statePool, forcingPool, timeLevel call mpas_pool_get_array(forcingPool, 'avgTotalFreshWaterTemperatureFlux', avgTotalFreshWaterTemperatureFlux) call mpas_pool_get_dimension(forcingPool, 'nCells', nCells) + call mpas_pool_get_dimension(forcingPool, 'nVertLevels', nVertLevels) call mpas_pool_get_dimension(forcingPool, & 'index_avgTemperatureSurfaceValue', & index_temperaturePtr) @@ -349,6 +372,7 @@ subroutine ocn_time_average_coupled_accumulate(statePool, forcingPool, timeLevel call mpas_pool_get_array(forcingPool, 'avgLandIceTracerTransferVelocities', avgLandIceTracerTransferVelocities) call mpas_pool_get_array(forcingPool, 'avgEffectiveDensityInLandIce', avgEffectiveDensityInLandIce) + !$omp parallel !$omp do schedule(runtime) do iCell = 1, nCells @@ -416,6 +440,41 @@ subroutine ocn_time_average_coupled_accumulate(statePool, forcingPool, timeLevel end if + if (trim(config_glc_thermal_forcing_coupling_mode) == '2d') then + call mpas_pool_get_array(forcingPool, 'avgThermalForcingAtCritDepth', avgThermalForcingAtCritDepth) + call mpas_pool_get_subpool(statePool, 'tracers', tracersPool) + call mpas_pool_get_array(tracersPool, 'activeTracers', activeTracers, 2) + call mpas_pool_get_dimension(tracersPool, 'index_temperature', indexTemperature) + call mpas_pool_get_dimension(tracersPool, 'index_salinity', indexSalinity) + + + ! find vertical level that is just above the critical depth reference level + ! this does not account for depression due to ice shelf cavities or sea ice + iLevelCritDepth = nVertLevels ! default to deepest layer if we don't find the desired depth + do iLevel = 1, nVertLevels + if(refBottomDepth(iLevel) > config_2d_thermal_forcing_depth) then + iLevelCritDepth = iLevel + exit + end if + end do + !$omp parallel + !$omp do schedule(runtime) + ! calculate thermal forcing at identified level for each cell + do iCell = 1, nCells + ! ignore cells that are too shallow + if (iLevelCritDepth <= maxLevelCell(iCell)) then + ! this uses the level shallower than the reference level. could interpolate instead + ! note: assuming no LandIce cavity, but we may want to support that + freezingTemp = ocn_freezing_temperature(salinity=activeTracers(indexSalinity, iLevelCritDepth, iCell), & + pressure=pressure(iLevelCritDepth, iCell), inLandIceCavity=.false.) + avgThermalForcingAtCritDepth(iCell) = ( avgThermalForcingAtCritDepth(iCell) * nAccumulatedCoupled & + + activeTracers(indexTemperature, iLevelCritDepth, iCell) - freezingTemp ) / ( nAccumulatedCoupled + 1) + end if + end do + !$omp end do + !$omp end parallel + endif + ! accumulate BGC coupling fields if necessary if (config_use_ecosysTracers) then diff --git a/components/mpas-ocean/src/shared/mpas_ocn_time_varying_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_time_varying_forcing.F index 8e64618a27ad..2afa3064050f 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_time_varying_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_time_varying_forcing.F @@ -258,13 +258,21 @@ subroutine atmospheric_forcing(streamManager, domain, simulationClock)!{{{ character(len=StrKIND) :: timeStamp - real (kind=RKIND) :: dtSim + real (kind=RKIND) :: dtSim, dtSimReverse + + real (kind=RKIND), pointer :: forcingTimeIncrement + + type (mpas_pool_type), pointer :: forcingPool integer :: err + call mpas_pool_get_subpool(domain % blocklist % structs, 'forcing', forcingPool) + call mpas_pool_get_array(forcingPool, 'forcingTimeIncrement', forcingTimeIncrement) + ! convert config_dt to real call mpas_set_timeInterval(timeStepESMF, timeString=config_dt,ierr=err) - call mpas_get_timeInterval(timeStepESMF, dt=dtSim) + dtSim = forcingTimeIncrement + dtSimReverse = -dtSim ! use the forcing layer to get data call MPAS_forcing_get_forcing(& @@ -277,8 +285,18 @@ subroutine atmospheric_forcing(streamManager, domain, simulationClock)!{{{ forcingGroupHead, & ! forcingGroupHead "ocn_atmospheric_forcing", & ! forcingGroupName currentForcingTime) ! forcingTime - !call mpas_get_time(curr_time=currentForcingTime, dateTimeString=timeStamp, ierr=err) - !call mpas_log_write('Forcing time ' // trim(timeStamp)) + call mpas_get_time(curr_time=currentForcingTime, dateTimeString=timeStamp, ierr=err) + !call mpas_log_write('Forcing time for atmospheric forcing' // trim(timeStamp)) + + call mpas_advance_forcing_clock(forcingGroupHead, dtSimReverse) + + call MPAS_forcing_get_forcing_time(& + forcingGroupHead, & ! forcingGroupHead + "ocn_atmospheric_forcing", & ! forcingGroupName + currentForcingTime) ! forcingTime + + call mpas_get_time(curr_time=currentForcingTime, dateTimeString=timeStamp, ierr=err) + !call mpas_log_write('Forcing time reversed for atmospheric forcing' // trim(timeStamp)) ! perform post forcing block => domain % blocklist @@ -332,7 +350,10 @@ subroutine post_atmospheric_forcing(block)!{{{ windStressCoefficient, & rhoAir, & ramp, & - windStressCoefficientLimit + windStressCoefficientLimit, & + t + + real(kind=RKIND), pointer :: forcingTimeIncrement integer, pointer :: & nCells @@ -354,12 +375,14 @@ subroutine post_atmospheric_forcing(block)!{{{ call MPAS_pool_get_array(timeVaryingForcingPool, "atmosPressure", atmosPressure) call MPAS_pool_get_array(forcingPool, "atmosphericPressure", atmosphericPressure) + call mpas_pool_get_array(forcingPool, 'forcingTimeIncrement', forcingTimeIncrement) rhoAir = 1.225_RKIND windStressCoefficientLimit = 0.0035_RKIND if (daysSinceStartOfSim >= config_time_varying_atmospheric_forcing_ramp_delay) then - ramp = tanh((2.0_RKIND*(daysSinceStartOfSim-config_time_varying_atmospheric_forcing_ramp_delay)) & + t = (daysSinceStartOfSim*86400_RKIND + forcingTimeIncrement)/86400.0_RKIND + ramp = tanh((2.0_RKIND*(t-config_time_varying_atmospheric_forcing_ramp_delay)) & /config_time_varying_atmospheric_forcing_ramp) else ramp = 0.0_RKIND @@ -483,11 +506,13 @@ subroutine land_ice_forcing(streamManager, domain, simulationClock)!{{{ type (block_type), pointer :: block -! character(len=StrKIND), pointer :: config_dt + character(len=StrKIND) :: timeStamp type (MPAS_timeInterval_type) :: timeStepESMF - real (kind=RKIND) :: dtSim + real (kind=RKIND) :: dtSim, dtSimReverse + + real (kind=RKIND), pointer :: forcingTimeIncrement integer :: err @@ -512,14 +537,41 @@ subroutine land_ice_forcing(streamManager, domain, simulationClock)!{{{ integer :: & iCell + call mpas_pool_get_subpool(domain % blocklist % structs, 'forcing', forcingPool) + call mpas_pool_get_array(forcingPool, 'forcingTimeIncrement', forcingTimeIncrement) + ! convert config_dt to real call mpas_set_timeInterval(timeStepESMF, timeString=config_dt,ierr=err) call mpas_get_timeInterval(timeStepESMF, dt=dtSim) + dtSim = forcingTimeIncrement + dtSimReverse = -dtSim + ! use the forcing layer to get data - call mpas_forcing_get_forcing(forcingGroupHead, "ocn_land_ice_forcing", streamManager, dtSim) + call MPAS_forcing_get_forcing(& + forcingGroupHead, & ! forcingGroupHead + "ocn_land_ice_forcing", & ! forcingGroupName + streamManager, & ! streamManager + dtSim) ! dt + + call MPAS_forcing_get_forcing_time(& + forcingGroupHead, & ! forcingGroupHead + "ocn_land_ice_forcing", & ! forcingGroupName + currentForcingTime) ! forcingTime + + call mpas_get_time(curr_time=currentForcingTime, dateTimeString=timeStamp, ierr=err) + call mpas_log_write('Forcing time for land ice forcing' // trim(timeStamp)) + + + call mpas_advance_forcing_clock(forcingGroupHead, dtSimReverse) + + call MPAS_forcing_get_forcing_time(& + forcingGroupHead, & ! forcingGroupHead + "ocn_land_ice_forcing", & ! forcingGroupName + currentForcingTime) ! forcingTime - call mpas_forcing_get_forcing_time(forcingGroupHead, "ocn_land_ice_forcing", currentForcingTime) + call mpas_get_time(curr_time=currentForcingTime, dateTimeString=timeStamp, ierr=err) + call mpas_log_write('Forcing time reversed for land ice forcing' // trim(timeStamp)) block => domain % blocklist do while (associated(block)) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tracer_advection_shared.F b/components/mpas-ocean/src/shared/mpas_ocn_tracer_advection_shared.F index 2d14c961fba1..3439776094de 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tracer_advection_shared.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tracer_advection_shared.F @@ -24,10 +24,12 @@ module ocn_tracer_advection_shared use mpas_derived_types use mpas_hash use mpas_sort + use mpas_constants use mpas_geometry_utils use mpas_log use ocn_config + use ocn_constants use ocn_mesh implicit none @@ -370,7 +372,6 @@ subroutine computeDerivTwo(derivTwo, err)!{{{ xec, yec, zec, &! arc bisection coords thetae_tmp, &! angle xv1, xv2, yv1, yv2, zv1, zv2, &! vertex cart coords - pii, &! pi math constant length_scale, &! length scale cos2t, costsint, sin2t ! trig function temps @@ -413,7 +414,6 @@ subroutine computeDerivTwo(derivTwo, err)!{{{ ! Initialize derivTwo and pi - pii = 2.*asin(1.0) derivTwo(:,:,:) = 0.0_RKIND do iCell = 1, nCellsAll @@ -461,9 +461,9 @@ subroutine computeDerivTwo(derivTwo, err)!{{{ end do if ( zc(1) == 1.0_RKIND) then - theta_abs(iCell) = pii/2.0_RKIND + theta_abs(iCell) = pi/2.0_RKIND else - theta_abs(iCell) = pii/2.0_RKIND & + theta_abs(iCell) = pi/2.0_RKIND & - mpas_sphere_angle( xc(1), yc(1), zc(1), & xc(2), yc(2), zc(2), & 0.0_RKIND, 0.0_RKIND, 1.0_RKIND) @@ -509,7 +509,7 @@ subroutine computeDerivTwo(derivTwo, err)!{{{ angle_2d(i) = angleEdge(edgesOnCell(i,iCell)) iEdge = edgesOnCell(i,iCell) if ( iCell .ne. cellsOnEdge(1,iEdge)) & - angle_2d(i) = angle_2d(i) - pii + angle_2d(i) = angle_2d(i) - pi xp(i) = dcEdge(edgesOnCell(i,iCell)) * cos(angle_2d(i)) yp(i) = dcEdge(edgesOnCell(i,iCell)) * sin(angle_2d(i)) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_vel_forcing_topographic_wave_drag.F b/components/mpas-ocean/src/shared/mpas_ocn_vel_forcing_topographic_wave_drag.F index 8f641ca4f31c..a024ec97e51b 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_vel_forcing_topographic_wave_drag.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_vel_forcing_topographic_wave_drag.F @@ -199,7 +199,7 @@ subroutine ocn_vel_forcing_topographic_wave_drag_init(err)!{{{ call mpas_log_write(" Topographic wave drag scheme is: Jayne and St. Laurent") call mpas_log_write("") tensorScheme = .false. - kappa = pii/10000.0_RKIND + kappa = pi/10000.0_RKIND do iEdge = 1, nEdgesAll kmax = maxLevelEdgeTop(iEdge) @@ -241,7 +241,7 @@ subroutine ocn_vel_forcing_topographic_wave_drag_init(err)!{{{ topographicWaveDrag(iEdge) = topographicWaveDragCoeff * gam & * bed_slope_edges(iEdge)**2 * Nbar * Nb & - / (8.0_RKIND * omegaM2 * pii**2) + / (8.0_RKIND * omegaM2 * pi**2) endif enddo else if (config_topographic_wave_drag_scheme.EQ."LGF") then @@ -256,7 +256,7 @@ subroutine ocn_vel_forcing_topographic_wave_drag_init(err)!{{{ topographicWaveDrag(iEdge) = topographicWaveDragCoeff & * (sqrt((topo_buoyancy_N1B(iEdge)**2 - omegaM2**2) & * (topo_buoyancy_N1V(iEdge)**2 - omegaM2**2))) & - / (4.0_RKIND*pii*omegaM2) + / (4.0_RKIND*pi*omegaM2) normalCoeffTWD(iEdge) = (lonGradEdge(iEdge)**2) & * (cos(angleEdge(iEdge))**2) + (latGradEdge(iEdge)**2)*(sin(angleEdge(iEdge))**2) & - 2.0_RKIND*latGradEdge(iEdge)*lonGradEdge(iEdge)*sin(angleEdge(iEdge))*(cos(angleEdge(iEdge))) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_vel_hmix_leith.F b/components/mpas-ocean/src/shared/mpas_ocn_vel_hmix_leith.F index a6f9ec5c073c..db3e78e9b881 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_vel_hmix_leith.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_vel_hmix_leith.F @@ -159,7 +159,7 @@ subroutine ocn_vel_hmix_leith_tend(div, relVort, tend, err)!{{{ dcEdgeInv = 1.0_RKIND / dcEdge(iEdge) dvEdgeInv = 1.0_RKIND / dvEdge(iEdge) - visc2tmp = (leithParam*dxLeith*meshScalingDel2(iEdge)/pii)**3 + visc2tmp = (leithParam*dxLeith*meshScalingDel2(iEdge)/pi)**3 do k = minLevelEdgeBot(iEdge), maxLevelEdgeTop(iEdge) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_vel_self_attraction_loading.F b/components/mpas-ocean/src/shared/mpas_ocn_vel_self_attraction_loading.F index 90d1a4e4630d..a1c7a65452f8 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_vel_self_attraction_loading.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_vel_self_attraction_loading.F @@ -863,8 +863,8 @@ subroutine ocn_vel_self_attraction_loading_init(domain,err)!{{{ ! Pre-compute sin and cos of latCell (co-latitude) values allocate(sinLatCell(nCellsAll), cosLatCell(nCellsAll)) do iCell = 1,nCellsAll - sinLatCell(iCell) = sin(0.5_RKIND*pii-latCell(iCell)) - cosLatCell(iCell) = cos(0.5_RKIND*pii-latCell(iCell)) + sinLatCell(iCell) = sin(0.5_RKIND*pi-latCell(iCell)) + cosLatCell(iCell) = cos(0.5_RKIND*pi-latCell(iCell)) enddo ! Calculate blocking indices @@ -2526,7 +2526,7 @@ subroutine associatedLegendrePolynomials(n, m, startIdx, endIdx, l, pmnm2, pmnm1 if (n == m) then do iCell = startIdx,endIdx - pmnm2(iCell) = sqrt(1.0_RKIND/(4.0_RKIND*pii))*sinLatCell(iCell)**m + pmnm2(iCell) = sqrt(1.0_RKIND/(4.0_RKIND*pi))*sinLatCell(iCell)**m do i = 1,m pmnm2(iCell) = pmnm2(iCell)*sqrt(real(2*i+1,RKIND)/real(2*i,RKIND)) enddo diff --git a/components/mpas-ocean/src/shared/mpas_ocn_vel_tidal_potential.F b/components/mpas-ocean/src/shared/mpas_ocn_vel_tidal_potential.F index 211538ac0a73..bd3d3298246c 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_vel_tidal_potential.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_vel_tidal_potential.F @@ -80,6 +80,8 @@ module ocn_vel_tidal_potential tidalConstituentAstronomical, &! tidalConstituentNodalPhase ! + real (kind=RKIND), pointer :: forcingTimeIncrement + real (kind=RKIND), dimension(:,:), pointer :: & latitudeFunction @@ -273,8 +275,8 @@ subroutine ocn_compute_tidal_potential_forcing(err)!{{{ err = 0 if (tidalPotentialOff) return - ramp = tanh((2.0_RKIND*daysSinceStartOfSim)/tidalPotRamp) - t = daysSinceStartOfSim*86400.0_RKIND + t = daysSinceStartOfSim*86400.0_RKIND + forcingTimeIncrement + ramp = tanh((2.0_RKIND*t/86400.0_RKIND)/tidalPotRamp) do iCell = 1, nCellsAll tidalPotEta(iCell) = 0.0_RKIND @@ -282,7 +284,7 @@ subroutine ocn_compute_tidal_potential_forcing(err)!{{{ !*** Compute eta by summing all constituent contributions do jCon = 1, nTidalConstituents - period = 2.0_RKIND*pii/tidalConstituentFrequency(jCon) + period = 2.0_RKIND*pi/tidalConstituentFrequency(jCon) nCycles = real(int(t/period),RKIND) targ = tidalConstituentFrequency(jCon)*(t - nCycles*period) + & tidalConstituentNodalPhase(jCon) + & @@ -416,6 +418,9 @@ subroutine ocn_vel_tidal_potential_init(domain,err)!{{{ call mpas_pool_get_array(forcingPool, & 'tidalPotentialLatitudeFunction', & latitudeFunction) + call mpas_pool_get_array(forcingPool, & + 'forcingTimeIncrement', & + forcingTimeIncrement) call mpas_set_time(refTime, & dateTimeString=config_tidal_potential_reference_time) @@ -583,7 +588,7 @@ subroutine tidal_constituent_factors(constituentList,nTidalConstituents,refTime, !h = 280.4661_RKIND + 0.98564736_RKIND*T; !p = 83.3535_RKIND + 0.11140353_RKIND*T; !N = 125.0445_RKIND - 0.05295377_RKIND*T; - !N = N*pii/180.0_RKIND + !N = N*pi/180.0_RKIND !! M2 !tidalConstituentAstronomical(j) = 2.0_RKIND*h - 2.0_RKIND*s @@ -611,7 +616,7 @@ subroutine tidal_constituent_factors(constituentList,nTidalConstituents,refTime, !tidalConstituentNodalAmplitude(j) = 1.009_RKIND + 0.187_RKIND*cos(N) - deg2rad = pii/180.0_RKIND + deg2rad = pi/180.0_RKIND T = adjust_angle(180.0_RKIND + real(refHour,RKIND)*(360.0_RKIND/24.0_RKIND)) do j = 1,nTidalConstituents @@ -742,8 +747,8 @@ subroutine orbit(year,julianDay,hour, & real (kind=RKIND) :: deg2rad,rad2deg real (kind=RKIND) :: NRad,pRad,IRad,nuRad,xiRad,nupRad,nup2Rad - deg2rad = pii/180.0_RKIND - rad2deg = 180.0_RKIND/pii + deg2rad = pi/180.0_RKIND + rad2deg = 180.0_RKIND/pi x = int((real(year,RKIND)-1901.0_RKIND)/4.0_RKIND) yr = real(year,RKIND) - 1900.0_RKIND diff --git a/components/mpas-ocean/src/shared/mpas_ocn_vmix.F b/components/mpas-ocean/src/shared/mpas_ocn_vmix.F index 68ef9d82a6bf..cabdf86ba112 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_vmix.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_vmix.F @@ -27,6 +27,7 @@ module ocn_vmix use mpas_timer use ocn_mesh + use ocn_diagnostics use mpas_constants use ocn_constants use ocn_config @@ -202,6 +203,11 @@ subroutine ocn_vmix_coefs(meshPool, statePool, forcingPool, scratchPool, err, ti !$omp end parallel #endif + if(config_use_cvmix_kpp .or. config_use_gotm) then + call ocn_compute_mixing_input_fields(statePool, forcingPool,& + meshPool, timeLevel) + end if + #ifdef MPAS_OPENACC !$acc update host(vertViscTopOfEdge, vertDiffTopOfCell) #endif diff --git a/components/mpas-ocean/src/shared/mpas_ocn_vmix_cvmix.F b/components/mpas-ocean/src/shared/mpas_ocn_vmix_cvmix.F index fcede65e420c..08042ae12947 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_vmix_cvmix.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_vmix_cvmix.F @@ -64,7 +64,7 @@ module ocn_vmix_cvmix type(cvmix_shear_params_type) :: cvmix_shear_params type(cvmix_tidal_params_type) :: cvmix_tidal_params - logical :: cvmixOn, cvmixConvectionOn, cvmixKPPOn + logical :: lnonzero_surf_nonlocal, cvmixOn, cvmixConvectionOn, cvmixKPPOn real (kind=RKIND) :: backgroundVisc, backgroundDiff integer :: cvmixBackgroundChoice ! user choice of cvmix background scheme @@ -141,6 +141,9 @@ subroutine ocn_vmix_coefs_cvmix_build(meshPool, statePool, forcingPool, err, tim integer, dimension(:), pointer :: & maxLevelCell, minLevelCell, nEdgesOnCell, maxLevelEdgeTop, minLevelEdgeBot + integer, dimension(:), pointer :: landIceMask + integer :: landIceMaskValue + real (kind=RKIND), dimension(:), pointer :: & latCell, lonCell, bottomDepth, fCell, & ssh, dcEdge, dvEdge, areaCell, iceFraction, & @@ -241,6 +244,7 @@ subroutine ocn_vmix_coefs_cvmix_build(meshPool, statePool, forcingPool, err, tim ! set pointers for fields related to ocean forcing state ! call mpas_pool_get_array(forcingPool, 'iceFraction', iceFraction) + call mpas_pool_get_array(forcingPool, 'landIceMask', landIceMask) call mpas_pool_get_array(forcingPool, 'windSpeed10m', windSpeed10m) call mpas_pool_get_array(forcingPool, 'windStressZonal', windStressZonal) call mpas_pool_get_array(forcingPool, 'windStressMeridional', windStressMeridional) @@ -345,8 +349,8 @@ subroutine ocn_vmix_coefs_cvmix_build(meshPool, statePool, forcingPool, err, tim ! specify geometry/location cvmix_variables % SeaSurfaceHeight = ssh(iCell) cvmix_variables % Coriolis = fCell(iCell) - cvmix_variables % lat = latCell(iCell) * 180.0_RKIND / 3.14_RKIND - cvmix_variables % lon = lonCell(iCell) * 180.0_RKIND / 3.14_RKIND + cvmix_variables % lat = latCell(iCell) * 180.0_RKIND / pi + cvmix_variables % lon = lonCell(iCell) * 180.0_RKIND / pi ! fill vertical position of column ! CVMix assume top of ocean is at z=0, so building all z-coordinate data based on layerThickness @@ -479,18 +483,25 @@ subroutine ocn_vmix_coefs_cvmix_build(meshPool, statePool, forcingPool, err, tim Nsqr_iface(k:maxLevelCell(iCell)+1) = Nsqr_iface(k-1) ! compute Langmuir number and Langmuir enhancement factor - if (config_cvmix_kpp_use_theory_wave .and. iceFraction(iCell) .lt. 0.05_RKIND) then - langmuirNumber(iCell) = sqrt(surfaceFrictionVelocity(iCell) / ( & + if (associated(landIceMask)) then + landIceMaskValue = landIceMask(iCell) + else + landIceMaskValue = 0 + endif + + if ( landIceMaskValue .eq. 0 .and. iceFraction(iCell) .lt. 0.05_RKIND) then + if (config_cvmix_kpp_use_theory_wave) then + langmuirNumber(iCell) = sqrt(surfaceFrictionVelocity(iCell) / ( & cvmix_kpp_ustokes_SL_model(windSpeed10m(iCell), & boundaryLayerDepth(iCell), & cvmix_global_params)+1e-15_RKIND) ) - langmuirEnhancementFactor = & + langmuirEnhancementFactor = & cvmix_kpp_EFactor_model(windSpeed10m(iCell), & surfaceFrictionVelocity(iCell), & boundaryLayerDepth(iCell), & cvmix_global_params) - else if (config_cvmix_kpp_use_active_wave .and. iceFraction(iCell) .lt. 0.05_RKIND) then - call ocn_stokes_drift_langmuir_number(windStressZonal(iCell), & + else if (config_cvmix_kpp_use_active_wave) then + call ocn_stokes_drift_langmuir_number(windStressZonal(iCell), & windStressMeridional(iCell), & surfaceFrictionVelocity(iCell), & significantWaveHeight(iCell), & @@ -499,9 +510,10 @@ subroutine ocn_vmix_coefs_cvmix_build(meshPool, statePool, forcingPool, err, tim stokesDriftMeridionalWavenumber(:,iCell), & alphaAngle, & langmuirNumber(iCell)) - call ocn_stokes_drift_kpp_enhancement_factor(alphaAngle, & + call ocn_stokes_drift_kpp_enhancement_factor(alphaAngle, & langmuirNumber(iCell), & langmuirEnhancementFactor) + endif else ! arbitrarily large Langmuir number langmuirNumber(iCell) = 1.e10_RKIND @@ -678,14 +690,21 @@ subroutine ocn_vmix_coefs_cvmix_build(meshPool, statePool, forcingPool, err, tim OBL_depth = boundaryLayerDepth(iCell) ) ! update Langmuir enhancement factor - if (config_cvmix_kpp_use_theory_wave .and. iceFraction(iCell) .lt. 0.05_RKIND) then - langmuirEnhancementFactor = & + if (associated(landIceMask)) then + landIceMaskValue = landIceMask(iCell) + else + landIceMaskValue = 0 + endif + + if (landIceMaskValue .eq. 0 .and. iceFraction(iCell) .lt. 0.05_RKIND) then + if (config_cvmix_kpp_use_theory_wave) then + langmuirEnhancementFactor = & cvmix_kpp_EFactor_model(windSpeed10m(iCell), & surfaceFrictionVelocity(iCell), & boundaryLayerDepth(iCell), & cvmix_global_params) - else if (config_cvmix_kpp_use_active_wave .and. iceFraction(iCell) .lt. 0.05_RKIND) then - call ocn_stokes_drift_langmuir_number(windStressZonal(iCell), & + else if (config_cvmix_kpp_use_active_wave) then + call ocn_stokes_drift_langmuir_number(windStressZonal(iCell), & windStressMeridional(iCell), & surfaceFrictionVelocity(iCell), & significantWaveHeight(iCell), & @@ -694,13 +713,13 @@ subroutine ocn_vmix_coefs_cvmix_build(meshPool, statePool, forcingPool, err, tim stokesDriftMeridionalWavenumber(:,iCell), & alphaAngle, & langmuirNumber(iCell)) - call ocn_stokes_drift_kpp_enhancement_factor(alphaAngle, & - langmuirNumber(iCell), & - langmuirEnhancementFactor) + call ocn_stokes_drift_kpp_enhancement_factor(alphaAngle, & + langmuirNumber(iCell), & + langmuirEnhancementFactor) + endif else langmuirEnhancementFactor = 1.0_RKIND - end if - + endif ! call mpas_timer_start('cvmix coeffs kpp', .false.) call cvmix_coeffs_kpp( & Mdiff_out = cvmix_variables % Mdiff_iface(minLevelCell(iCell):maxLevelCell(iCell)+1), & @@ -1042,19 +1061,22 @@ subroutine ocn_vmix_cvmix_init(domain,err)!{{{ ! initialize KPP boundary layer scheme ! if (config_use_cvmix_kpp) then - if(config_cvmix_kpp_matching.eq."MatchBoth") then - call mpas_log_write( & - "Use of option MatchBoth is discouraged, use SimpleShapes instead", & - MPAS_LOG_WARN) - elseif(.not. config_cvmix_kpp_matching.eq."SimpleShapes") then + if(.not. config_cvmix_kpp_matching.eq."SimpleShapes" .and. & + .not. config_cvmix_kpp_matching.eq."MatchBoth" .and. & + .not. config_cvmix_kpp_matching.eq."ParabolicNonLocal") then call mpas_log_write( & "Unknown value for config_cvmix_kpp_matching., supported values are:" // & - " SimpleShapes or MatchBoth", & + " SimpleShapes or MatchBoth or ParabolicNonLocal", & MPAS_LOG_CRIT) err = 1 return endif + lnonzero_surf_nonlocal = .false. + if(config_cvmix_kpp_matching .eq."ParabolicNonLocal") then + lnonzero_surf_nonlocal = .true. + end if + if (trim(config_cvmix_kpp_langmuir_mixing_opt) .ne. "NONE" .and. & trim(config_cvmix_kpp_langmuir_mixing_opt) .ne. "LWF16" .and. & trim(config_cvmix_kpp_langmuir_mixing_opt) .ne. "RWHGK16") then @@ -1081,13 +1103,15 @@ subroutine ocn_vmix_cvmix_init(domain,err)!{{{ call cvmix_init_kpp ( & ri_crit = config_cvmix_kpp_criticalBulkRichardsonNumber, & interp_type = config_cvmix_kpp_interpolationOMLType, & - interp_type2 = config_cvmix_kpp_interpolationOMLType, & + interp_type2 = 'LMD94', & lEkman = config_cvmix_kpp_EkmanOBL, & lMonOb = config_cvmix_kpp_MonObOBL, & MatchTechnique = config_cvmix_kpp_matching, & surf_layer_ext = config_cvmix_kpp_surface_layer_extent, & langmuir_mixing_str = config_cvmix_kpp_langmuir_mixing_opt, & langmuir_entrainment_str = config_cvmix_kpp_langmuir_entrainment_opt, & + lnoDGat1 = .true., & + lnonzero_surf_nonlocal = lnonzero_surf_nonlocal, & lenhanced_diff = config_cvmix_kpp_use_enhanced_diff) endif diff --git a/components/mpas-seaice/bld/build-namelist b/components/mpas-seaice/bld/build-namelist index 3d7ce2c619f8..87831cc94b03 100755 --- a/components/mpas-seaice/bld/build-namelist +++ b/components/mpas-seaice/bld/build-namelist @@ -946,6 +946,7 @@ if ($iceberg_mode eq 'data') { } else { add_default($nl, 'config_use_data_icebergs', 'val'=>"false"); } +add_default($nl, 'config_scale_dib_by_removed_ice_runoff'); add_default($nl, 'config_salt_flux_coupling_type'); add_default($nl, 'config_ice_ocean_drag_coefficient'); diff --git a/components/mpas-seaice/bld/build-namelist-section b/components/mpas-seaice/bld/build-namelist-section index d2866ee4c813..90dd6ce39f21 100644 --- a/components/mpas-seaice/bld/build-namelist-section +++ b/components/mpas-seaice/bld/build-namelist-section @@ -451,6 +451,7 @@ add_default($nl, 'config_ocean_heat_transfer_type'); add_default($nl, 'config_sea_freezing_temperature_type'); add_default($nl, 'config_ocean_surface_type'); add_default($nl, 'config_use_data_icebergs'); +add_default($nl, 'config_scale_dib_by_removed_ice_runoff'); add_default($nl, 'config_salt_flux_coupling_type'); add_default($nl, 'config_ice_ocean_drag_coefficient'); diff --git a/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml b/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml index d62797de4518..8128c268c6e0 100644 --- a/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml +++ b/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml @@ -31,6 +31,7 @@ 120.0 60.0 900.0 +1800.0 'noleap' '2000-01-01_00:00:00' 'none' @@ -87,6 +88,7 @@ 85.0 85.0 85.0 +85.0 75.0 85.0 85.0 @@ -102,6 +104,7 @@ -85.0 -85.0 -85.0 +-85.0 -85.0 -85.0 -85.0 @@ -170,6 +173,7 @@ 1 1 2 +1 true true 120 @@ -452,6 +456,7 @@ 'mushy' 'free' false +false 'constant' 0.00536 diff --git a/components/mpas-seaice/bld/namelist_files/namelist_definition_mpassi.xml b/components/mpas-seaice/bld/namelist_files/namelist_definition_mpassi.xml index efbaef07cb9d..d19f3f4eaad5 100644 --- a/components/mpas-seaice/bld/namelist_files/namelist_definition_mpassi.xml +++ b/components/mpas-seaice/bld/namelist_files/namelist_definition_mpassi.xml @@ -1077,7 +1077,7 @@ Default: Defined in namelist_defaults.xml -Use the humic matter tracer +Use the humic (refractory dissolved organic matter) tracer Valid values: true or false Default: Defined in namelist_defaults.xml @@ -1949,7 +1949,7 @@ Default: Defined in namelist_defaults.xml -Transport type of humics +Transport type of humics (refractory dissolved organic matter) Valid values: -1 = entirely in the mobile phase; 0 = retention dominated; 1 = release dominated; 0.5 = equal but rapid exchange; 2 = equal but slow exchange Default: Defined in namelist_defaults.xml @@ -2686,6 +2686,14 @@ Valid values: true or false Default: Defined in namelist_defaults.xml + +Whether to scale data iceberg fluxes by the running mean of removed ice runoff + +Valid values: true or false +Default: Defined in namelist_defaults.xml + + Type of salt flux to ocean method diff --git a/components/mpas-seaice/cime_config/buildnml b/components/mpas-seaice/cime_config/buildnml index c38ca08a31da..5df71d69b1a8 100755 --- a/components/mpas-seaice/cime_config/buildnml +++ b/components/mpas-seaice/cime_config/buildnml @@ -300,7 +300,7 @@ def buildnml(case, caseroot, compname): grid_prefix = 'mpassi.IcoswISC30E3r5' decomp_date = '20231120' decomp_prefix = 'partitions/mpas-seaice.graph.info.' - data_iceberg_file = 'Iceberg_Climatology_Merino.IcoswISC30E3r5.20231120.nc' + data_iceberg_file = 'Iceberg_Climatology_Merino.IcoswISC30E3r5.20240805.nc' dust_iron_file = 'ecoForcingSurfaceMonthly+WetDryDustSolFrac.IcoswISC30E3r5.20240511.nc' if ice_ic_mode == 'spunup': if iceberg_mode == 'data': @@ -330,6 +330,16 @@ def buildnml(case, caseroot, compname): logger.warning("WARNING: The specified compset is requesting seaice ICs spunup from a G-case") logger.warning(" But no file available for this grid.") + elif ice_grid == 'SOwISC12to30E3r3': + grid_date = '20240829' + grid_prefix = 'mpassi.SOwISC12to30E3r3' + decomp_date = '20240829' + decomp_prefix = 'partitions/mpas-seaice.graph.info.' + data_iceberg_file = 'Iceberg_Climatology_Merino.SOwISC12to30E3r3.20241017.nc' + if ice_ic_mode == 'spunup': + grid_date = '20240829' + grid_prefix = 'mpassi.SOwISC12to30E3r3.rstFromG-chrysalis' + elif ice_grid == 'ICOS10': grid_date = '211015' grid_prefix = 'seaice.ICOS10' @@ -658,6 +668,7 @@ def buildnml(case, caseroot, compname): lines.append(' input_interval="none" >') lines.append('') lines.append(' ') + lines.append(' ') lines.append(' ') lines.append('') lines.append('') @@ -924,22 +935,41 @@ def buildnml(case, caseroot, compname): lines.append(' ') if ice_bgc == 'ice_bgc': - lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') - lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') - lines.append(' ') - lines.append(' ') - lines.append(' ') lines.append(' ') lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') @@ -999,15 +1029,35 @@ def buildnml(case, caseroot, compname): lines.append(' ') lines.append(' ') lines.append(' ') - lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') - lines.append(' ') - lines.append(' ') - lines.append(' ') lines.append(' ') lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') diff --git a/components/mpas-seaice/driver/ice_comp_mct.F b/components/mpas-seaice/driver/ice_comp_mct.F index 3c579aee77af..f9d79bab96f2 100644 --- a/components/mpas-seaice/driver/ice_comp_mct.F +++ b/components/mpas-seaice/driver/ice_comp_mct.F @@ -194,7 +194,9 @@ subroutine ice_init_mct( EClock, cdata_i, x2i_i, i2x_i, NLFilename )!{{{ type (MPAS_TimeInterval_Type) :: alarmTimeStep type (block_type), pointer :: block - type (MPAS_Pool_Type), pointer :: shortwave + type (MPAS_Pool_Type), pointer :: & + shortwave, & + berg_forcing logical :: exists logical :: verbose_taskmap_output ! true then use verbose task-to-node mapping format @@ -241,15 +243,18 @@ subroutine ice_init_mct( EClock, cdata_i, x2i_i, i2x_i, NLFilename )!{{{ integer :: size_list, index_list type(mct_string) :: mctOStr ! character(CXX) :: mct_field, modelStr -#endif +#endif #endif - logical, pointer :: tempLogicalConfig + logical, pointer :: & + tempLogicalConfig, & + config_scale_dib_by_removed_ice_runoff character(len=StrKIND), pointer :: tempCharConfig real (kind=RKIND), pointer :: tempRealConfig real(kind=RKIND), pointer :: & - dayOfNextShortwaveCalculation ! needed for CESM like coupled simulations + dayOfNextShortwaveCalculation, & ! needed for CESM like coupled simulations + runningMeanRemovedIceRunoff ! the area integrated, running mean of removed ice runoff from the ocean interface subroutine xml_stream_parser(xmlname, mgr_p, comm, ierr) bind(c) @@ -505,6 +510,9 @@ end subroutine xml_stream_get_attributes end if + call MPAS_pool_get_config(domain % configs, "config_scale_dib_by_removed_ice_runoff", & + config_scale_dib_by_removed_ice_runoff) + ! Setup MPASSI simulation clock ierr = domain % core % setup_clock(domain % clock, domain % configs) if ( ierr /= 0 ) then @@ -636,6 +644,15 @@ end subroutine xml_stream_get_attributes ! Determine coupling type call seq_infodata_GetData(infodata, cpl_seq_option=cpl_seq_option) + if (config_scale_dib_by_removed_ice_runoff) then + ! independent of space so should be no need to loop over blocks + block => domain % blocklist + call MPAS_pool_get_subpool(block % structs, "berg_forcing", berg_forcing) + call MPAS_pool_get_array(berg_forcing, "runningMeanRemovedIceRunoff", & + runningMeanRemovedIceRunoff) + call seq_infodata_GetData(infodata, rmean_rmv_ice_runoff=runningMeanRemovedIceRunoff ) + end if + ! Determine time of next atmospheric shortwave calculation block => domain % blocklist do while (associated(block)) @@ -730,7 +747,7 @@ end subroutine xml_stream_get_attributes allocate (x2i_im(lsize, nrecv) ) i2x_im = 0._r8 x2i_im = 0._r8 - ! define tags according to the seq_flds_i2x_fields + ! define tags according to the seq_flds_i2x_fields ! also zero them out tagtype = 1 ! dense, double numco = 1 ! one value per cell / entity @@ -763,7 +780,7 @@ end subroutine xml_stream_get_attributes endif #endif - + !----------------------------------------------------------------------- ! @@ -827,10 +844,10 @@ end subroutine xml_stream_get_attributes else if (trim(tempCharConfig) == "column_package") then call seaice_column_coupling_prep(domain) endif ! config_column_physics_type - + call MPAS_pool_get_config(domain % configs, "config_use_floe_size_distribution", tempLogicalConfig) if (tempLogicalConfig) then - call mpas_log_write('FloeSizeDistribution coming online soon. Turn FSD off for now.', MPAS_LOG_CRIT) + call mpas_log_write('FloeSizeDistribution coming online soon. Turn FSD off for now.', MPAS_LOG_CRIT) endif !----------------------------------------------------------------------- ! @@ -854,7 +871,7 @@ end subroutine xml_stream_get_attributes ! ! get intial state from driver ! - call ice_import_mct(x2i_i, errorCode) + call ice_import_mct(x2i_i, errorCode) if (errorCode /= 0) then call mpas_log_write('Error in ice_import_mct', MPAS_LOG_CRIT) endif @@ -862,7 +879,7 @@ end subroutine xml_stream_get_attributes #ifdef HAVE_MOAB #ifdef MOABCOMP - mpicom_moab = mpicom_i ! save it for run method + mpicom_moab = mpicom_i ! save it for run method ! loop over all fields in seq_flds_x2i_fields call mct_list_init(temp_list ,seq_flds_x2i_fields) size_list=mct_list_nitem (temp_list) @@ -879,8 +896,8 @@ end subroutine xml_stream_get_attributes #endif call ice_import_moab(Eclock) -#endif - +#endif + currTime = mpas_get_clock_time(domain % clock, MPAS_NOW, ierr) call mpas_get_time(curr_time=currTime, dateTimeString=timeStamp, ierr=ierr) @@ -1151,7 +1168,8 @@ subroutine ice_run_mct( EClock, cdata_i, x2i_i, i2x_i)!{{{ ! Variable related to MPASSI type (block_type), pointer :: block type (MPAS_Pool_type), pointer :: & - shortwave + shortwave, & + berg_forcing real (kind=RKIND) :: current_wallclock_time type (MPAS_Time_Type) :: currTime @@ -1159,13 +1177,16 @@ subroutine ice_run_mct( EClock, cdata_i, x2i_i, i2x_i)!{{{ type (MPAS_timeInterval_type) :: timeStep integer :: ierr, streamDirection, iam logical :: streamActive, debugOn - logical, pointer :: config_write_output_on_startup + logical, pointer :: & + config_write_output_on_startup, & + config_scale_dib_by_removed_ice_runoff logical, save :: first=.true. character (len=StrKIND), pointer :: & config_restart_timestamp_name, & config_column_physics_type real(kind=RKIND), pointer :: & - dayOfNextShortwaveCalculation ! needed for CESM like coupled simulations + dayOfNextShortwaveCalculation, & ! needed for CESM like coupled simulations + runningMeanRemovedIceRunoff ! the area integrated, running mean of removed ice runoff from the ocean #ifdef MOABCOMP real(r8) :: difference @@ -1173,7 +1194,7 @@ subroutine ice_run_mct( EClock, cdata_i, x2i_i, i2x_i)!{{{ integer :: size_list, index_list, ent_type type(mct_string) :: mctOStr ! character(CXX) :: mct_field, modelStr, tagname -#endif +#endif iam = domain % dminfo % my_proc_id @@ -1187,8 +1208,10 @@ subroutine ice_run_mct( EClock, cdata_i, x2i_i, i2x_i)!{{{ mpas_log_info => domain % logInfo if (debugOn) call mpas_log_write("=== Beginning ice_run_mct ===") - call mpas_pool_get_config(domain % configs, 'config_restart_timestamp_name', config_restart_timestamp_name) + call MPAS_pool_get_config(domain % configs, 'config_restart_timestamp_name', config_restart_timestamp_name) call MPAS_pool_get_config(domain % configs, "config_column_physics_type", config_column_physics_type) + call MPAS_pool_get_config(domain % configs, "config_scale_dib_by_removed_ice_runoff", & + config_scale_dib_by_removed_ice_runoff) ! Setup log information. call shr_file_getLogUnit (shrlogunit) @@ -1225,6 +1248,15 @@ subroutine ice_run_mct( EClock, cdata_i, x2i_i, i2x_i)!{{{ ! Post coupling calls block => domain % blocklist + + if (config_scale_dib_by_removed_ice_runoff) then + ! independent of space so should be no need to loop over blocks + call MPAS_pool_get_subpool(block % structs, "berg_forcing", berg_forcing) + call MPAS_pool_get_array(berg_forcing, "runningMeanRemovedIceRunoff", & + runningMeanRemovedIceRunoff) + call seq_infodata_GetData(infodata, rmean_rmv_ice_runoff=runningMeanRemovedIceRunoff ) + end if + do while (associated(block)) ! Determine time of next atmospheric shortwave calculation @@ -2998,8 +3030,8 @@ subroutine frazil_mass(freezingPotential, frazilMassFlux, seaSurfaceSalinity) qi0new, & vi0new - call MPAS_pool_get_config(domain % configs, "config_thermodynamics_type", config_thermodynamics_type) - call MPAS_pool_get_config(domain % configs, "config_column_physics_type", config_column_physics_type) + call MPAS_pool_get_config(domain % configs, "config_thermodynamics_type", config_thermodynamics_type) + call MPAS_pool_get_config(domain % configs, "config_column_physics_type", config_column_physics_type) if (freezingPotential > 0.0_RKIND) then @@ -3479,15 +3511,15 @@ subroutine ice_import_moab(Eclock)!{{{ ! o dhdx -- ocn surface slope, zonal ! o dhdy -- ocn surface slope, meridional ! o lwdn -- downward lw heat flux -! o rain -- prec: liquid -! o snow -- prec: frozen +! o rain -- prec: liquid +! o snow -- prec: frozen ! o swndr -- sw: nir direct downward ! o swvdr -- sw: vis direct downward ! o swndf -- sw: nir diffuse downward ! o swvdf -- sw: vis diffuse downward ! o swnet -- sw: net ! o q -- ocn frazil heat flux(+) / melt potential(-) -! o frazil -- ocn frazil mass flux +! o frazil -- ocn frazil mass flux ! o bcphidry -- Black Carbon hydrophilic dry deposition flux ! o bcphodry -- Black Carbon hydrophobic dry deposition flux ! o bcphiwet -- Black Carbon hydrophilic wet deposition flux @@ -3502,10 +3534,10 @@ subroutine ice_import_moab(Eclock)!{{{ ! o dstdry2 -- Size 2 dust -- dry deposition flux ! o dstdry3 -- Size 3 dust -- dry deposition flux ! o dstdry4 -- Size 4 dust -- dry deposition flux -! +! ! The following fields are sometimes received from the coupler, ! depending on model options: -! +! ! o algae1 -- ! o algae2 -- ! o algae3 -- @@ -3530,7 +3562,7 @@ subroutine ice_import_moab(Eclock)!{{{ ! o zaer4 -- ! o zaer5 -- ! o zaer6 -- -! +! !----------------------------------------------------------------------- ! ! !REVISION HISTORY: @@ -3550,7 +3582,7 @@ subroutine ice_import_moab(Eclock)!{{{ character (len=StrKIND) :: & label, & message - + integer :: & i,n @@ -3687,10 +3719,10 @@ subroutine ice_import_moab(Eclock)!{{{ if (ierr > 0 ) then write(iceLogUnit,*) 'Fail to write ice state ' endif -#endif -!----------------------------------------------------------------------- +#endif +!----------------------------------------------------------------------- ! -! zero out padded cells +! zero out padded cells ! !----------------------------------------------------------------------- diff --git a/components/mpas-seaice/src/Registry.xml b/components/mpas-seaice/src/Registry.xml index 7c842ca2f3c4..26559e4aa49d 100644 --- a/components/mpas-seaice/src/Registry.xml +++ b/components/mpas-seaice/src/Registry.xml @@ -866,7 +866,7 @@ icepack_name="tr_bgc_PON" /> @@ -948,77 +948,77 @@ possible_values="positive real number" icepack_name="frazil_scav" /> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -1473,32 +1474,32 @@ possible_values="-1 = entirely in the mobile phase; 0 = retention dominated; 1 = release dominated; 0.5 = equal but rapid exchange; 2 = equal but slow exchange" icepack_name="zaerotype_dust4" /> - - - - - - - - - - - - - - - - - - + + + - - + + @@ -2701,17 +2711,17 @@ - - - - - - - - - - - + + + + + + + + + + + @@ -2719,6 +2729,7 @@ + @@ -2885,11 +2896,13 @@ @@ -3422,241 +3481,325 @@ + + + + + + - + + + @@ -5304,6 +5478,8 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -5887,6 +6264,7 @@ diff --git a/components/mpas-seaice/src/model_forward/mpas_seaice_core_interface.F b/components/mpas-seaice/src/model_forward/mpas_seaice_core_interface.F index 4b128d0c534d..6240ea9f7057 100644 --- a/components/mpas-seaice/src/model_forward/mpas_seaice_core_interface.F +++ b/components/mpas-seaice/src/model_forward/mpas_seaice_core_interface.F @@ -8,6 +8,7 @@ module seaice_core_interface use seaice_analysis_driver use mpas_log, only: mpas_log_write + implicit none public contains @@ -546,10 +547,10 @@ end subroutine setup_packages_column_physics!}}} ! routine setup_packages_bergs ! !> \brief Setup icebergs package - !> \author Darin Comeau - !> \date 19 May 2017 - !> \details This routine is intended to set the icebergs package PkgBergs - !> as active/deactive based on the namelist option config_use_bergs. + !> \author Darin Comeau, Xylar Asay-Davis + !> \date August 2024 + !> \details This routine is intended to set the icebergs packages PkgBergs + !> and pkgScaledDIB as active/deactive based on the namelist options. ! !----------------------------------------------------------------------- @@ -561,10 +562,13 @@ subroutine setup_packages_bergs(configPool, packagePool, ierr)!{{{ ! icebergs package logical, pointer :: & - config_use_data_icebergs + config_use_data_icebergs, & + config_scale_dib_by_removed_ice_runoff + logical, pointer :: & - pkgBergsActive + pkgBergsActive, & + pkgScaledDIBActive ierr = 0 @@ -576,6 +580,10 @@ subroutine setup_packages_bergs(configPool, packagePool, ierr)!{{{ call MPAS_pool_get_package(packagePool, "pkgBergsActive", pkgBergsActive) pkgBergsActive = config_use_data_icebergs + call MPAS_pool_get_config(configPool, "config_scale_dib_by_removed_ice_runoff", config_scale_dib_by_removed_ice_runoff) + call MPAS_pool_get_package(packagePool, "pkgScaledDIBActive", pkgScaledDIBActive) + pkgScaledDIBActive = config_scale_dib_by_removed_ice_runoff + end subroutine setup_packages_bergs!}}} !*********************************************************************** diff --git a/components/mpas-seaice/src/seaice.cmake b/components/mpas-seaice/src/seaice.cmake index 8b72de8ec7d7..5d314fd7594d 100644 --- a/components/mpas-seaice/src/seaice.cmake +++ b/components/mpas-seaice/src/seaice.cmake @@ -144,6 +144,7 @@ set(SEAICE_MODEL_FORWARD ) list(APPEND RAW_SOURCES ${SEAICE_MODEL_FORWARD}) list(APPEND DISABLE_QSMP ${SEAICE_MODEL_FORWARD}) +list(APPEND NOOPT_FILES "core_seaice/icepack/columnphysics/icepack_shortwave_data.F90") # Generate core input handle_st_nl_gen("namelist.seaice" "streams.seaice stream_list.seaice. listed" ${CORE_INPUT_DIR} ${CORE_BLDDIR}) diff --git a/components/mpas-seaice/src/shared/mpas_seaice_forcing.F b/components/mpas-seaice/src/shared/mpas_seaice_forcing.F index ca0c4c5a962f..868189757005 100644 --- a/components/mpas-seaice/src/shared/mpas_seaice_forcing.F +++ b/components/mpas-seaice/src/shared/mpas_seaice_forcing.F @@ -3318,6 +3318,9 @@ subroutine get_data_iceberg_fluxes(domain) berg_forcing, & berg_fluxes + logical, pointer :: & + config_scale_dib_by_removed_ice_runoff + integer, pointer :: & nCellsSolve @@ -3326,14 +3329,36 @@ subroutine get_data_iceberg_fluxes(domain) bergFreshwaterFlux, & ! iceberg freshwater flux for ocean (kg/m^2/s) bergLatentHeatFlux ! iceberg latent heat flux for ocean (J/m^2/s) + real(kind=RKIND), pointer :: & + areaIntegAnnMeanDataIcebergIceShelfFreshwaterFlux, & ! area integrated, annual mean freshwater flux from icebegs and ice shelves (kg/s) + runningMeanRemovedIceRunoff ! the area integrated, running mean of removed ice runoff (kg/s) + integer :: & iCell + real(kind=RKIND) :: & + scaling + ! dc including as parameters here so as not to create new namelist options real(kind=RKIND), parameter :: & specificHeatFreshIce = 2106.0_RKIND, & ! specific heat of fresh ice J * kg^-1 * K^-1 bergTemperature = -4.0_RKIND ! iceberg temperature, assumed constant + + call MPAS_pool_get_config(domain % configs, "config_scale_dib_by_removed_ice_runoff", & + config_scale_dib_by_removed_ice_runoff) + + if (config_scale_dib_by_removed_ice_runoff) then + block => domain % blocklist + call MPAS_pool_get_subpool(block % structs, "berg_forcing", berg_forcing) + call mpas_pool_get_array(berg_forcing, 'runningMeanRemovedIceRunoff', runningMeanRemovedIceRunoff) + call mpas_pool_get_array(berg_forcing, 'areaIntegAnnMeanDataIcebergIceShelfFreshwaterFlux', & + areaIntegAnnMeanDataIcebergIceShelfFreshwaterFlux) + scaling = runningMeanRemovedIceRunoff / areaIntegAnnMeanDataIcebergIceShelfFreshwaterFlux + else + scaling = 1.0_RKIND + end if + block => domain % blocklist do while (associated(block)) @@ -3349,8 +3374,8 @@ subroutine get_data_iceberg_fluxes(domain) do iCell = 1, nCellsSolve - bergFreshwaterFlux(iCell) = bergFreshwaterFluxData(iCell) - bergLatentHeatFlux(iCell) = -bergFreshwaterFluxData(iCell) * & + bergFreshwaterFlux(iCell) = scaling * bergFreshwaterFluxData(iCell) + bergLatentHeatFlux(iCell) = -scaling * bergFreshwaterFluxData(iCell) * & (seaiceLatentHeatMelting - specificHeatFreshIce*bergTemperature) enddo diff --git a/components/mpas-seaice/src/shared/mpas_seaice_icepack.F b/components/mpas-seaice/src/shared/mpas_seaice_icepack.F index 36899b4b10d7..9dc6f06ec342 100644 --- a/components/mpas-seaice/src/shared/mpas_seaice_icepack.F +++ b/components/mpas-seaice/src/shared/mpas_seaice_icepack.F @@ -3923,7 +3923,11 @@ subroutine column_biogeochemistry(domain) abortFlag = icepack_warnings_aborted() call get_cice_tracer_array_category(block, ciceTracerObject, & - tracerArrayCategory, iCell, setGetPhysicsTracers, setGetBGCTracers) + tracerArrayCategory, iCell, setGetPhysicsTracers, setGetBGCTracers) + + if (config_use_vertical_tracers) & + call define_total_biogeochemistry_array_cell(block, ciceTracerObject, totalVerticalBiologyIce(:,iCell), & + totalVerticalBiologySnow(:,iCell), iCell) if (checkCarbon) then call seaice_total_carbon_content_category(block,totalCarbonCatFinal(:,iCell),iceAreaCategory(1,:,:),iceVolumeCategory(1,:,:),iCell) @@ -8648,10 +8652,13 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace nzAerosols type(MPAS_pool_type), pointer :: & - tracers_aggregate + tracers_aggregate, & + biogeochemistry real(kind=RKIND), dimension(:), pointer :: & - brineFractionCell + brineFractionCell, & + carbonToNitrogenRatioAlgae, & + carbonToNitrogenRatioDON real(kind=RKIND), dimension(:,:), pointer :: & skeletalAlgaeConcCell, & @@ -8699,14 +8706,21 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace verticalParticulateIronIceCell, & verticalDissolvedIronIceCell, & verticalAerosolsIceCell, & - verticalAerosolsSnowCell + verticalAerosolsSnowCell, & + verticalDOCLabileIceCell, & + verticalAlgaeTotalCarbonIceCell, & + verticalBCTotalIceCell, & + verticalDustTotalIceCell, & + verticalBCTotalSnowCell, & + verticalDustTotalSnowCell integer :: & iBioTracers, & iBioCount, & iLayers, & iIceCount, & - iSnowCount + iSnowCount, & + nAeroType call MPAS_pool_get_config(block % configs, "config_use_skeletal_biochemistry", config_use_skeletal_biochemistry) call MPAS_pool_get_config(block % configs, "config_use_vertical_biochemistry", config_use_vertical_biochemistry) @@ -8736,6 +8750,7 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace call MPAS_pool_get_dimension(block % dimensions, "nDissolvedIron", nDissolvedIron) call MPAS_pool_get_subpool(block % structs, "tracers_aggregate", tracers_aggregate) + call MPAS_pool_get_subpool(block % structs, "biogeochemistry", biogeochemistry) call MPAS_pool_get_array(tracers_aggregate, "skeletalAlgaeConcCell", skeletalAlgaeConcCell) call MPAS_pool_get_array(tracers_aggregate, "skeletalDOCConcCell", skeletalDOCConcCell) @@ -8784,6 +8799,14 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace call MPAS_pool_get_array(tracers_aggregate, "verticalAerosolsSnowCell", verticalAerosolsSnowCell) call MPAS_pool_get_array(tracers_aggregate, "verticalSalinityCell", verticalSalinityCell) call MPAS_pool_get_array(tracers_aggregate, "brineFractionCell", brineFractionCell) + call MPAS_pool_get_array(tracers_aggregate, "verticalAlgaeTotalCarbonIceCell", verticalAlgaeTotalCarbonIceCell) + call MPAS_pool_get_array(tracers_aggregate, "verticalDOCLabileIceCell", verticalDOCLabileIceCell) + call MPAS_pool_get_array(tracers_aggregate, "verticalBCTotalIceCell", verticalBCTotalIceCell) + call MPAS_pool_get_array(tracers_aggregate, "verticalDustTotalIceCell", verticalDustTotalIceCell) + call MPAS_pool_get_array(tracers_aggregate, "verticalDustTotalSnowCell", verticalDustTotalSnowCell) + call MPAS_pool_get_array(tracers_aggregate, "verticalBCTotalSnowCell", verticalBCTotalSnowCell) + call MPAS_pool_get_array(biogeochemistry, "carbonToNitrogenRatioAlgae", carbonToNitrogenRatioAlgae) + call MPAS_pool_get_array(biogeochemistry, "carbonToNitrogenRatioDON", carbonToNitrogenRatioDON) ! biogeochemistry ! brine height fraction @@ -8866,12 +8889,15 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace ! algal nitrogen do iBioTracers = 1, nAlgae iIceCount = (iBioTracers-1)*nBioLayersP1 + verticalAlgaeTotalCarbonIceCell(:,iCell) = 0.0_RKIND do iLayers = 1,nBioLayersP1 iBiocount = iBiocount + 1 verticalAlgaeConcCell(iBioCount,iCell) = & tracerArrayCell(tracerObject % index_algaeConc(iBioTracers)+iLayers-1) verticalAlgaeIceCell(iLayers+iIceCount,iCell) = verticalAlgaeConcCell(iBioCount,iCell) + verticalAlgaeTotalCarbonIceCell(iLayers,iCell) = verticalAlgaeTotalCarbonIceCell(iLayers,iCell) + & + carbonToNitrogenRatioAlgae(iBioTracers) * verticalAlgaeConcCell(iBioCount,iCell) enddo do iLayers = nBioLayersP1+1,nBioLayersP3 iBiocount = iBiocount + 1 @@ -8894,6 +8920,7 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace if (config_use_carbon) then iBioCount = 0 + verticalDOCLabileIceCell(:,iCell) = 0.0_RKIND ! DOC do iBioTracers = 1, nDOC @@ -8904,6 +8931,8 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace verticalDOCConcCell(iBioCount,iCell) = & tracerArrayCell(tracerObject % index_DOCConc(iBioTracers) + iLayers-1) verticalDOCIceCell(iLayers+iIceCount,iCell) = verticalDOCConcCell(iBioCount,iCell) + verticalDOCLabileIceCell(iLayers, iCell) = verticalDOCLabileIceCell(iLayers, iCell) + & + verticalDOCConcCell(iBioCount,iCell) enddo do iLayers = nBioLayersP1+1,nBioLayersP3 iBioCount = iBioCount + 1 @@ -8942,6 +8971,8 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace verticalDONConcCell(iBioCount,iCell) = & tracerArrayCell(tracerObject % index_DONConc(iBioTracers) + iLayers-1) verticalDONIceCell(iLayers+iIceCount,iCell) = verticalDONConcCell(iBioCount,iCell) + verticalDOCLabileIceCell(iLayers, iCell) = verticalDOCLabileIceCell(iLayers, iCell) + & + verticalDONConcCell(iBioCount,iCell) * carbonToNitrogenRatioDON(iBioTracers) enddo do iLayers = nBioLayersP1+1,nBioLayersP3 iBioCount = iBioCount + 1 @@ -9058,15 +9089,25 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace ! black carbon and dust aerosols if (config_use_zaerosols) then iBioCount = 0 + verticalDustTotalIceCell(:,iCell) = 0.0_RKIND + verticalBCTotalIceCell(:,iCell) = 0.0_RKIND + verticalDustTotalSnowCell(:,iCell) = 0.0_RKIND + verticalBCTotalSnowCell(:,iCell) = 0.0_RKIND + do iBioTracers = 1, nzAerosols iIceCount = (iBioTracers-1)*nBioLayersP1 iSnowCount = (iBioTracers-1)*2 - + nAeroType = 1 + if (iBioTracers .gt. 2) nAeroType = 0 do iLayers = 1,nBioLayersP1 iBioCount = iBioCount + 1 verticalAerosolsConcCell(iBioCount,iCell) = & tracerArrayCell(tracerObject % index_verticalAerosolsConc(iBioTracers)+iLayers-1) verticalAerosolsIceCell(iLayers+iIceCount,iCell) = verticalAerosolsConcCell(iBioCount,iCell) + verticalBCTotalIceCell(iLayers, iCell) = verticalBCTotalIceCell(iLayers, iCell) + nAeroType * & + verticalAerosolsConcCell(iBioCount,iCell) + verticalDustTotalIceCell(iLayers, iCell) = verticalDustTotalIceCell(iLayers, iCell) + (1-nAeroType) * & + verticalAerosolsConcCell(iBioCount,iCell) enddo do iLayers = nBioLayersP1+1,nBioLayersP3 iBioCount = iBioCount + 1 @@ -9074,6 +9115,10 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace tracerArrayCell(tracerObject % index_verticalAerosolsConc(iBioTracers)+iLayers-1) verticalAerosolsSnowCell(iLayers-nBioLayersP1+iSnowCount,iCell) = & verticalAerosolsConcCell(iBioCount,iCell) + verticalBCTotalSnowCell(iLayers-nBioLayersP1, iCell) = verticalBCTotalSnowCell(iLayers-nBioLayersP1, iCell) + & + nAeroType * verticalAerosolsConcCell(iBioCount,iCell) + verticalDustTotalSnowCell(iLayers-nBioLayersP1,iCell) = verticalDustTotalSnowCell(iLayers-nBioLayersP1,iCell) + & + (1-nAeroType) * verticalAerosolsConcCell(iBioCount,iCell) enddo enddo endif @@ -9088,6 +9133,274 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace end subroutine get_cice_biogeochemistry_tracer_array_cell +!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +! +! define_total_biogeochemistry_array_cell +! +!> \brief +!> \author Nicole Jeffery, LANL +!> \date 16rd September 2024 +!> \details +!> +! Defines total ice and snow arrays for biogeochemistry and aerosols diagnostics +! +!----------------------------------------------------------------------- + + subroutine define_total_biogeochemistry_array_cell(block, tracerObject, totalVerticalBioIceCell, & + totalVerticalBioSnowCell, iCell) + + type(block_type), intent(in) :: & + block + + type(ciceTracerObjectType), intent(in) :: & + tracerObject + + real(kind=RKIND), dimension(:), intent(in):: & + totalVerticalBioIceCell, & + totalVerticalBioSnowCell + + integer, intent(in) :: & + iCell + + logical, pointer :: & + config_use_vertical_biochemistry, & + config_use_nitrate, & + config_use_carbon, & + config_use_ammonium, & + config_use_silicate, & + config_use_DMS, & + config_use_nonreactive, & + config_use_humics, & + config_use_DON, & + config_use_iron, & + config_use_zaerosols + + integer, pointer :: & + nAlgae, & + nDOC, & + nDIC, & + nDON, & + nParticulateIron, & + nDissolvedIron, & + nzAerosols + + type(MPAS_pool_type), pointer :: & + biogeochemistry + + real(kind=RKIND), dimension(:), pointer :: & + totalVerticalDiatomIce, & + totalVerticalSmallPlanktonIce, & + totalVerticalPhaeocystisIce, & + totalVerticalNitrateIce, & + totalVerticalPolysaccharidsIce, & + totalVerticalLipidsIce, & + totalVerticalDICIce, & + totalVerticalAmmoniumIce, & + totalVerticalSilicateIce, & + totalVerticalDMSPpIce, & + totalVerticalDMSPdIce, & + totalVerticalDMSIce, & + totalVerticalNonreactiveIce, & + totalVerticalHumicsIce, & + totalVerticalProteinsIce, & + totalVerticalDissolvedIronIce, & + totalVerticalParticulateIronIce, & + totalVerticalDustIce, & + totalVerticalDustSnow, & + totalVerticalBC1Ice, & + totalVerticalBC1Snow, & + totalVerticalBC2Ice, & + totalVerticalBC2Snow, & + totalVerticalBCIce, & + totalVerticalBCSnow, & + totalVerticalDissolvedIronSnow, & + totalVerticalAlgaeCarbonIce, & + totalVerticalDOCLabileIce, & + carbonToNitrogenRatioAlgae, & + carbonToNitrogenRatioDON + + call MPAS_pool_get_config(block % configs, "config_use_vertical_biochemistry", config_use_vertical_biochemistry) + call MPAS_pool_get_config(block % configs, "config_use_nitrate", config_use_nitrate) + call MPAS_pool_get_config(block % configs, "config_use_carbon", config_use_carbon) + call MPAS_pool_get_config(block % configs, "config_use_ammonium",config_use_ammonium) + call MPAS_pool_get_config(block % configs, "config_use_silicate",config_use_silicate) + call MPAS_pool_get_config(block % configs, "config_use_DMS",config_use_DMS) + call MPAS_pool_get_config(block % configs, "config_use_nonreactive",config_use_nonreactive) + call MPAS_pool_get_config(block % configs, "config_use_humics",config_use_humics) + call MPAS_pool_get_config(block % configs, "config_use_DON",config_use_DON) + call MPAS_pool_get_config(block % configs, "config_use_iron",config_use_iron) + call MPAS_pool_get_config(block % configs, "config_use_zaerosols",config_use_zaerosols) + + call MPAS_pool_get_dimension(block % dimensions, "nzAerosols", nzAerosols) + call MPAS_pool_get_dimension(block % dimensions, "nAlgae", nAlgae) + call MPAS_pool_get_dimension(block % dimensions, "nDOC", nDOC) + call MPAS_pool_get_dimension(block % dimensions, "nDIC", nDIC) + call MPAS_pool_get_dimension(block % dimensions, "nDON", nDON) + call MPAS_pool_get_dimension(block % dimensions, "nParticulateIron", nParticulateIron) + call MPAS_pool_get_dimension(block % dimensions, "nDissolvedIron", nDissolvedIron) + + call MPAS_pool_get_subpool(block % structs, "biogeochemistry", biogeochemistry) + + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDiatomIce", totalVerticalDiatomIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalSmallPlanktonIce", totalVerticalSmallPlanktonIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalPhaeocystisIce", totalVerticalPhaeocystisIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalPolysaccharidsIce", totalVerticalPolysaccharidsIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalLipidsIce", totalVerticalLipidsIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDICIce", totalVerticalDICIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalProteinsIce", totalVerticalProteinsIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalNitrateIce", totalVerticalNitrateIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalSilicateIce", totalVerticalSilicateIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalAmmoniumIce", totalVerticalAmmoniumIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDMSIce", totalVerticalDMSIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDMSPpIce", totalVerticalDMSPpIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDMSPdIce", totalVerticalDMSPdIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalNonreactiveIce", totalVerticalNonreactiveIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalHumicsIce", totalVerticalHumicsIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalParticulateIronIce", totalVerticalParticulateIronIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDissolvedIronIce", totalVerticalDissolvedIronIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDissolvedIronSnow", totalVerticalDissolvedIronSnow) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDustIce", totalVerticalDustIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDustSnow", totalVerticalDustSnow) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalBC1Ice", totalVerticalBC1Ice) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalBC1Snow", totalVerticalBC1Snow) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalBC2Ice", totalVerticalBC2Ice) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalBC2Snow", totalVerticalBC2Snow) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalBCIce", totalVerticalBCIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalBCSnow", totalVerticalBCSnow) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalAlgaeCarbonIce", totalVerticalAlgaeCarbonIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDOCLabileIce", totalVerticalDOCLabileIce) + call MPAS_pool_get_array(biogeochemistry, "carbonToNitrogenRatioAlgae", carbonToNitrogenRatioAlgae) + call MPAS_pool_get_array(biogeochemistry, "carbonToNitrogenRatioDON", carbonToNitrogenRatioDON) + + ! biogeochemistry + + if (config_use_vertical_biochemistry) then + + ! algal nitrogen + totalVerticalDiatomIce(iCell) = totalVerticalBioIceCell(tracerObject % index_algaeConcLayer(1)) + totalVerticalAlgaeCarbonIce(iCell) = totalVerticalDiatomIce(iCell) * carbonToNitrogenRatioAlgae(1) + + SELECT CASE (nAlgae) + CASE (2) + totalVerticalSmallPlanktonIce(iCell) = totalVerticalBioIceCell(tracerObject % index_algaeConcLayer(2)) + totalVerticalAlgaeCarbonIce(iCell) = totalVerticalAlgaeCarbonIce(iCell) + & + totalVerticalSmallPlanktonIce(iCell) * carbonToNitrogenRatioAlgae(2) + CASE (3) + totalVerticalSmallPlanktonIce(iCell) = totalVerticalBioIceCell(tracerObject % index_algaeConcLayer(2)) + totalVerticalPhaeocystisIce(iCell) = totalVerticalBioIceCell(tracerObject % index_algaeConcLayer(3)) + totalVerticalAlgaeCarbonIce(iCell) = totalVerticalAlgaeCarbonIce(iCell) + & + totalVerticalSmallPlanktonIce(iCell) * carbonToNitrogenRatioAlgae(2) + & + totalVerticalPhaeocystisIce(iCell) * carbonToNitrogenRatioAlgae(3) + END SELECT + end if + + ! nitrate + if (config_use_nitrate) & + totalVerticalNitrateIce(iCell) = totalVerticalBioIceCell(tracerObject % index_nitrateConcLayer) + + if (config_use_carbon) then + + ! DOC + totalVerticalDOCLabileIce(iCell) = 0.0_RKIND + SELECT CASE (nDOC) + CASE (1) + totalVerticalPolysaccharidsIce(iCell) = totalVerticalBioIceCell(tracerObject % index_DOCConcLayer(1)) + totalVerticalDOCLabileIce(iCell) = totalVerticalPolysaccharidsIce(iCell) + CASE (2) + totalVerticalPolysaccharidsIce(iCell) = totalVerticalBioIceCell(tracerObject % index_DOCConcLayer(1)) + totalVerticalLipidsIce(iCell) = totalVerticalBioIceCell(tracerObject % index_DOCConcLayer(1)) + totalVerticalDOCLabileIce(iCell) = totalVerticalPolysaccharidsIce(iCell) + totalVerticalLipidsIce(iCell) + END SELECT + + ! DIC + SELECT CASE (nDIC) + CASE (1) + totalVerticalDICIce(iCell) = totalVerticalBioIceCell(tracerObject % index_DICConcLayer(1)) + END SELECT + end if + + if (config_use_DON) then + + ! DON + SELECT CASE (nDON) + CASE (1) + totalVerticalProteinsIce(iCell) = totalVerticalBioIceCell(tracerObject % index_DONConcLayer(1)) + totalVerticalDOCLabileIce(iCell) = totalVerticalDOCLabileIce(iCell) + & + totalVerticalProteinsIce(iCell)* carbonToNitrogenRatioDON(1) + END SELECT + end if + + ! ammonium + if (config_use_ammonium) & + totalVerticalAmmoniumIce(iCell) = totalVerticalBioIceCell(tracerObject % index_ammoniumConcLayer) + + ! silicate + if (config_use_silicate) & + totalVerticalSilicateIce(iCell) = totalVerticalBioIceCell(tracerObject % index_silicateConcLayer) + + ! DMS, DMSPp, DMSPd + if (config_use_DMS) then + totalVerticalDMSIce(iCell) = totalVerticalBioIceCell(tracerObject % index_DMSConcLayer) + totalVerticalDMSPpIce(iCell) = totalVerticalBioIceCell(tracerObject % index_DMSPpConcLayer) + totalVerticalDMSPdIce(iCell) = totalVerticalBioIceCell(tracerObject % index_DMSPdConcLayer) + end if + + ! nonreactive + if (config_use_nonreactive) & + totalVerticalNonreactiveIce(iCell) = totalVerticalBioIceCell(tracerObject % index_nonreactiveConcLayer) + + ! humic material (refractory DOC) + if (config_use_humics) then + totalVerticalHumicsIce(iCell) = totalVerticalBioIceCell(tracerObject % index_humicsConcLayer) + end if + + if (config_use_iron) then + + ! ParticulateIron + SELECT CASE (nParticulateIron) + CASE (1) + totalVerticalParticulateIronIce(iCell) = totalVerticalBioIceCell(tracerObject % index_particulateIronConcLayer(1)) + END SELECT + + ! DissolvedIron + SELECT CASE (nDissolvedIron) + CASE (1) + totalVerticalDissolvedIronIce(iCell) = totalVerticalBioIceCell(tracerObject % index_dissolvedIronConcLayer(1)) + totalVerticalDissolvedIronSnow(iCell) = totalVerticalBioSnowCell(tracerObject % index_dissolvedIronConcLayer(1)) + END SELECT + + end if + + ! black carbon and dust aerosols + if (config_use_zaerosols) then + + SELECT CASE (nzAerosols) + CASE (1) + totalVerticalBC1Ice(iCell) = totalVerticalBioIceCell(tracerObject % index_verticalAerosolsConcLayer(1)) + totalVerticalBC1Snow(iCell) = totalVerticalBioSnowCell(tracerObject % index_verticalAerosolsConcLayer(1)) + totalVerticalBCIce(iCell) = totalVerticalBC1Ice(iCell) + totalVerticalBCSnow(iCell) = totalVerticalBC1Snow(iCell) + CASE (2) + totalVerticalBC1Ice(iCell) = totalVerticalBioIceCell(tracerObject % index_verticalAerosolsConcLayer(1)) + totalVerticalBC1Snow(iCell) = totalVerticalBioSnowCell(tracerObject % index_verticalAerosolsConcLayer(1)) + totalVerticalBC2Ice(iCell) = totalVerticalBioIceCell(tracerObject % index_verticalAerosolsConcLayer(2)) + totalVerticalBC2Snow(iCell) = totalVerticalBioSnowCell(tracerObject % index_verticalAerosolsConcLayer(2)) + totalVerticalBCIce(iCell) = totalVerticalBC1Ice(iCell) + totalVerticalBC2Ice(iCell) + totalVerticalBCSnow(iCell) = totalVerticalBC1Snow(iCell) + totalVerticalBC2Snow(iCell) + CASE (3) + totalVerticalBC1Ice(iCell) = totalVerticalBioIceCell(tracerObject % index_verticalAerosolsConcLayer(1)) + totalVerticalBC1Snow(iCell) = totalVerticalBioSnowCell(tracerObject % index_verticalAerosolsConcLayer(1)) + totalVerticalBC2Ice(iCell) = totalVerticalBioIceCell(tracerObject % index_verticalAerosolsConcLayer(2)) + totalVerticalBC2Snow(iCell) = totalVerticalBioSnowCell(tracerObject % index_verticalAerosolsConcLayer(2)) + totalVerticalDustIce(iCell) = totalVerticalBioIceCell(tracerObject % index_verticalAerosolsConcLayer(3)) + totalVerticalDustSnow(iCell) = totalVerticalBioSnowCell(tracerObject % index_verticalAerosolsConcLayer(3)) + totalVerticalBCIce(iCell) = totalVerticalBC1Ice(iCell) + totalVerticalBC2Ice(iCell) + totalVerticalBCSnow(iCell) = totalVerticalBC1Snow(iCell) + totalVerticalBC2Snow(iCell) + END SELECT + end if + + end subroutine define_total_biogeochemistry_array_cell + !----------------------------------------------------------------------- ! Init CICE parameters !----------------------------------------------------------------------- @@ -14689,7 +15002,8 @@ subroutine seaice_icepack_reinitialize_diagnostics_bgc(domain) type(MPAS_pool_type), pointer :: & biogeochemistryPool, & - diagnostics_biogeochemistryPool + diagnostics_biogeochemistryPool, & + tracers_aggregatePool ! biogeochemistry real(kind=RKIND), dimension(:), pointer :: & @@ -14699,7 +15013,35 @@ subroutine seaice_icepack_reinitialize_diagnostics_bgc(domain) ! zSalinityFlux, & !echmod deprecate ! zSalinityGDFlux, & !echmod deprecate totalChlorophyll, & - totalCarbonContentCell + totalCarbonContentCell, & + totalVerticalDiatomIce, & + totalVerticalSmallPlanktonIce, & + totalVerticalPhaeocystisIce, & + totalVerticalNitrateIce, & + totalVerticalPolysaccharidsIce, & + totalVerticalLipidsIce, & + totalVerticalDICIce, & + totalVerticalAmmoniumIce, & + totalVerticalSilicateIce, & + totalVerticalDMSPpIce, & + totalVerticalDMSPdIce, & + totalVerticalDMSIce, & + totalVerticalNonreactiveIce, & + totalVerticalHumicsIce, & + totalVerticalProteinsIce, & + totalVerticalDissolvedIronIce, & + totalVerticalParticulateIronIce, & + totalVerticalDustIce, & + totalVerticalDustSnow, & + totalVerticalBC1Ice, & + totalVerticalBC1Snow, & + totalVerticalBC2Ice, & + totalVerticalBC2Snow, & + totalVerticalBCIce, & + totalVerticalBCSnow, & + totalVerticalAlgaeCarbonIce, & + totalVerticalDOCLabileIce, & + totalVerticalDissolvedIronSnow real(kind=RKIND), dimension(:,:), pointer :: & oceanBioFluxes, & @@ -14709,7 +15051,12 @@ subroutine seaice_icepack_reinitialize_diagnostics_bgc(domain) totalVerticalBiologySnow, & bgridPorosityIceCell, & bgridSalinityIceCell, & - bgridTemperatureIceCell + bgridTemperatureIceCell, & + verticalBCTotalIceCell, & + verticalDustTotalIceCell, & + verticalBCTotalSnowCell, & + verticalDustTotalSnowCell + real(kind=RKIND), dimension(:,:,:), pointer :: & bioTracerShortwave @@ -14739,6 +15086,7 @@ subroutine seaice_icepack_reinitialize_diagnostics_bgc(domain) call MPAS_pool_get_subpool(block % structs, "biogeochemistry", biogeochemistryPool) call MPAS_pool_get_subpool(block % structs, "diagnostics_biogeochemistry", diagnostics_biogeochemistryPool) + call MPAS_pool_get_subpool(block % structs, "tracers_aggregate", tracers_aggregatePool) if (config_use_vertical_tracers) then call MPAS_pool_get_array(biogeochemistryPool, "primaryProduction", primaryProduction) @@ -14747,13 +15095,12 @@ subroutine seaice_icepack_reinitialize_diagnostics_bgc(domain) call MPAS_pool_get_array(diagnostics_biogeochemistryPool, "bgridSalinityIceCell", bgridSalinityIceCell) call MPAS_pool_get_array(diagnostics_biogeochemistryPool, "bgridPorosityIceCell", bgridPorosityIceCell) call MPAS_pool_get_array(diagnostics_biogeochemistryPool, "bgridTemperatureIceCell", bgridTemperatureIceCell) - - primaryProduction = 0.0_RKIND - totalChlorophyll = 0.0_RKIND - netSpecificAlgalGrowthRate = 0.0_RKIND - bgridSalinityIceCell = 0.0_RKIND - bgridPorosityIceCell = 0.0_RKIND - bgridTemperatureIceCell = 0.0_RKIND + primaryProduction = 0.0_RKIND + totalChlorophyll = 0.0_RKIND + netSpecificAlgalGrowthRate = 0.0_RKIND + bgridSalinityIceCell = 0.0_RKIND + bgridPorosityIceCell = 0.0_RKIND + bgridTemperatureIceCell = 0.0_RKIND end if @@ -14764,6 +15111,38 @@ subroutine seaice_icepack_reinitialize_diagnostics_bgc(domain) call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBiologyIce", totalVerticalBiologyIce) call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBiologySnow", totalVerticalBiologySnow) call MPAS_pool_get_array(biogeochemistryPool, "totalCarbonContentCell", totalCarbonContentCell) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDiatomIce", totalVerticalDiatomIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalSmallPlanktonIce", totalVerticalSmallPlanktonIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalPhaeocystisIce", totalVerticalPhaeocystisIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalPolysaccharidsIce", totalVerticalPolysaccharidsIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalLipidsIce", totalVerticalLipidsIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDICIce", totalVerticalDICIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalProteinsIce", totalVerticalProteinsIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalNitrateIce", totalVerticalNitrateIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalSilicateIce", totalVerticalSilicateIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalAmmoniumIce", totalVerticalAmmoniumIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDMSIce", totalVerticalDMSIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDMSPpIce", totalVerticalDMSPpIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDMSPdIce", totalVerticalDMSPdIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalNonreactiveIce", totalVerticalNonreactiveIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalHumicsIce", totalVerticalHumicsIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalParticulateIronIce", totalVerticalParticulateIronIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDissolvedIronIce", totalVerticalDissolvedIronIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDissolvedIronSnow", totalVerticalDissolvedIronSnow) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDustIce", totalVerticalDustIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDustSnow", totalVerticalDustSnow) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBC1Ice", totalVerticalBC1Ice) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBC1Snow", totalVerticalBC1Snow) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBC2Ice", totalVerticalBC2Ice) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBC2Snow", totalVerticalBC2Snow) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBCIce", totalVerticalBCIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBCSnow", totalVerticalBCSnow) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalAlgaeCarbonIce", totalVerticalAlgaeCarbonIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDOCLabileIce", totalVerticalDOCLabileIce) + call MPAS_pool_get_array(tracers_aggregatePool, "verticalBCTotalIceCell", verticalBCTotalIceCell) + call MPAS_pool_get_array(tracers_aggregatePool, "verticalDustTotalIceCell", verticalDustTotalIceCell) + call MPAS_pool_get_array(tracers_aggregatePool, "verticalDustTotalSnowCell", verticalDustTotalSnowCell) + call MPAS_pool_get_array(tracers_aggregatePool, "verticalBCTotalSnowCell", verticalBCTotalSnowCell) netBrineHeight = 0.0_RKIND @@ -14773,6 +15152,38 @@ subroutine seaice_icepack_reinitialize_diagnostics_bgc(domain) totalVerticalBiologyIce = 0.0_RKIND totalVerticalBiologySnow = 0.0_RKIND totalCarbonContentCell = 0.0_RKIND + totalVerticalDiatomIce = 0.0_RKIND + totalVerticalSmallPlanktonIce = 0.0_RKIND + totalVerticalPhaeocystisIce = 0.0_RKIND + totalVerticalPolysaccharidsIce = 0.0_RKIND + totalVerticalLipidsIce = 0.0_RKIND + totalVerticalDICIce = 0.0_RKIND + totalVerticalProteinsIce = 0.0_RKIND + totalVerticalNitrateIce = 0.0_RKIND + totalVerticalSilicateIce = 0.0_RKIND + totalVerticalAmmoniumIce = 0.0_RKIND + totalVerticalDMSIce = 0.0_RKIND + totalVerticalDMSPpIce = 0.0_RKIND + totalVerticalDMSPdIce = 0.0_RKIND + totalVerticalNonreactiveIce = 0.0_RKIND + totalVerticalHumicsIce = 0.0_RKIND + totalVerticalParticulateIronIce = 0.0_RKIND + totalVerticalDissolvedIronIce = 0.0_RKIND + totalVerticalDissolvedIronSnow = 0.0_RKIND + totalVerticalBC1Ice = 0.0_RKIND + totalVerticalBC2Ice = 0.0_RKIND + totalVerticalDustIce = 0.0_RKIND + totalVerticalBC1Snow = 0.0_RKIND + totalVerticalBC2Snow = 0.0_RKIND + totalVerticalBCSnow = 0.0_RKIND + totalVerticalBCIce = 0.0_RKIND + totalVerticalDustSnow = 0.0_RKIND + totalVerticalAlgaeCarbonIce = 0.0_RKIND + totalVerticalDOCLabileIce = 0.0_RKIND + verticalBCTotalIceCell = 0.0_RKIND + verticalDustTotalIceCell = 0.0_RKIND + verticalBCTotalSnowCell = 0.0_RKIND + verticalDustTotalSnowCell = 0.0_RKIND endif diff --git a/components/ww3/bld/build-namelist b/components/ww3/bld/build-namelist index 314949671c4f..092dab3ffba5 100755 --- a/components/ww3/bld/build-namelist +++ b/components/ww3/bld/build-namelist @@ -363,7 +363,7 @@ if ($NML_TYPE eq "ww3_grid") { add_default($nl, 'grid%dmin'); add_default($nl, 'unst%sf'); - add_default($nl, 'unst%filename', 'val'=>"'${DIN_LOC_ROOT}/wav/ww3/${WAV_GRID}.msh'"); + add_default($nl, 'unst%filename', 'val'=>"'${DIN_LOC_ROOT}/wav/ww3/${WAV_GRID}_rtd.msh'"); add_default($nl, 'unst%idf'); add_default($nl, 'unst%idla'); add_default($nl, 'unst%idfm'); @@ -398,8 +398,8 @@ if ($NML_TYPE eq "ww3_grid_nml") { add_default($nl, 'icedisp'); add_default($nl, 'rwndc'); - add_default($nl, 'uostfilelocal', 'val'=>"'${DIN_LOC_ROOT}/wav/ww3/obstructions_local.${WAV_GRID}${WAV_SPEC}.in'"); - add_default($nl, 'uostfileshadow', 'val'=>"'${DIN_LOC_ROOT}/wav/ww3/obstructions_shadow.${WAV_GRID}${WAV_SPEC}.in'"); + add_default($nl, 'uostfilelocal', 'val'=>"'${DIN_LOC_ROOT}/wav/ww3/obstructions_local.${WAV_GRID}${WAV_SPEC}.rtd.in'"); + add_default($nl, 'uostfileshadow', 'val'=>"'${DIN_LOC_ROOT}/wav/ww3/obstructions_shadow.${WAV_GRID}${WAV_SPEC}.rtd.in'"); } diff --git a/components/ww3/bld/namelist_files/namelist_defaults_ww3_grid.xml b/components/ww3/bld/namelist_files/namelist_defaults_ww3_grid.xml index e8bb1d42b243..0e4d46740870 100644 --- a/components/ww3/bld/namelist_files/namelist_defaults_ww3_grid.xml +++ b/components/ww3/bld/namelist_files/namelist_defaults_ww3_grid.xml @@ -11,9 +11,9 @@ variables. Values that depend on the model configuration use attributes to express the dependency. --> -1.07 +1.1 0.035 -50 +36 36 0.5 @@ -30,7 +30,7 @@ attributes to express the dependency. 450.0 30.0 -wQU225EC30to60E2r2 +wQU225Icos30E3r5 ww3_grid_namelists.nml UNST SPHE diff --git a/components/ww3/bld/namelist_files/namelist_defaults_ww3_grid_nml.xml b/components/ww3/bld/namelist_files/namelist_defaults_ww3_grid_nml.xml index 9573b482a45e..9ebe2b4e5059 100644 --- a/components/ww3/bld/namelist_files/namelist_defaults_ww3_grid_nml.xml +++ b/components/ww3/bld/namelist_files/namelist_defaults_ww3_grid_nml.xml @@ -158,8 +158,8 @@ attributes to express the dependency. -obstructions_local.glo_unst.in -obstructions_shadow.glo_unst.in +obstructions_local.rtd.in +obstructions_shadow.rtd.in 1.00 1.00 diff --git a/components/ww3/cime_config/buildlib_cmake b/components/ww3/cime_config/buildlib_cmake index 4f15ea2d58e3..42d92b2f0ad4 100755 --- a/components/ww3/cime_config/buildlib_cmake +++ b/components/ww3/cime_config/buildlib_cmake @@ -43,14 +43,20 @@ def buildlib(bldroot, installpath, case): # Define WW3 repository directories repodir = "{}/components/ww3/src".format(srcroot) modeldir = "{}/WW3/model".format(repodir) - - # TODO: these work dirs will have to be changed to live in the binary/build area. - # Achieving this will probably require a significant refactor of the ww3 infrastructure. - # Doing this stuff in-source not only clutters the repo, but also introduces potential race - # conditions if we were to try to build multiple ww3 cases simultaneously. - bindir = "{}/bin".format(modeldir) - exedir = "{}/exe".format(modeldir) - tmpdir = "{}/tmp".format(modeldir) + builddir = "{}/wav".format(exeroot) + + # work dirs are placed in the binary/build area. + bindir_source = "{}/bin".format(modeldir) + bindir = "{}/bin".format(builddir) + shutil.copytree(bindir_source, bindir) + auxdir_source = "{}/aux".format(modeldir) + auxdir = "{}/aux".format(builddir) + shutil.copytree(auxdir_source, auxdir) + ftndir_source = "{}/ftn".format(modeldir) + ftndir = "{}/ftn".format(builddir) + shutil.copytree(ftndir_source, ftndir) + + tmpdir = "{}/tmp".format(builddir) # Run w3_setup to create wwatch3.env file env_file = os.path.join(bindir, "wwatch3.env") @@ -77,7 +83,7 @@ def buildlib(bldroot, installpath, case): y """.format(sf90, scc, tmpdir)) - run_bld_cmd_ensure_logging("./w3_setup {} -s E3SM < w3_setup.inp".format(modeldir), logger, from_dir=bindir) + run_bld_cmd_ensure_logging("./w3_setup {} -s E3SM < w3_setup.inp".format(builddir), logger, from_dir=bindir) os.remove(inp_file) # Generate pre-processed WW3 source code @@ -85,7 +91,7 @@ y for exe in ww3_exe: run_bld_cmd_ensure_logging("./w3_source {}".format(exe), logger, from_dir=bindir) for exe in ww3_exe: - tarfile = "{}/work/{}.tar.gz".format(modeldir,exe) + tarfile = "{}/work/{}.tar.gz".format(builddir,exe) shutil.move(tarfile, tmpdir) run_bld_cmd_ensure_logging("tar -xzvf {}.tar.gz".format(exe), logger, from_dir=tmpdir) run_bld_cmd_ensure_logging("rm ww3_shel.F90", logger, from_dir=tmpdir) diff --git a/components/ww3/cime_config/buildnml b/components/ww3/cime_config/buildnml index 8e7cd2ea84ec..c43a521c29cc 100755 --- a/components/ww3/cime_config/buildnml +++ b/components/ww3/cime_config/buildnml @@ -55,7 +55,8 @@ def buildnml(case, caseroot, compname): "wQU225EC60to30sp50x36", "wQU225EC30to60E2r2sp50x36", "wQU225EC30to60E2r2sp36x36", - "wQU225EC30to60E2r2sp25x36") + "wQU225EC30to60E2r2sp25x36", + "wQU225Icos30E3r5sp36x36") expect((wav_grid+wav_spec) in wav_grid_supported, "Combination of WAV_GRID {} and WAV_SPEC {} is not supported in ww3. Choose from: '{}'".format(wav_grid,wav_spec,wav_grid_supported) ) #-------------------------------------------------------------------- @@ -64,10 +65,10 @@ def buildnml(case, caseroot, compname): with open(os.path.join(casebuild, "ww3.input_data_list"), "w") as input_list: - input_list.write("mesh = {}/wav/ww3/{}.msh\n".format(din_loc_root,wav_grid)) + input_list.write("mesh = {}/wav/ww3/{}_rtd.msh\n".format(din_loc_root,wav_grid)) input_list.write("stations = {}/wav/ww3/stations.txt\n".format(din_loc_root)) - input_list.write("uostfilelocal = {}/wav/ww3/obstructions_local.{}{}.in\n".format(din_loc_root,wav_grid,wav_spec)) - input_list.write("uostfileshadow = {}/wav/ww3/obstructions_shadow.{}{}.in\n".format(din_loc_root,wav_grid,wav_spec)) + input_list.write("uostfilelocal = {}/wav/ww3/obstructions_local.{}{}.rtd.in\n".format(din_loc_root,wav_grid,wav_spec)) + input_list.write("uostfileshadow = {}/wav/ww3/obstructions_shadow.{}{}.rtd.in\n".format(din_loc_root,wav_grid,wav_spec)) #-------------------------------------------------------------------- # Invoke ww3 build-namelist - output will go in $CASEBUILD/ww3conf diff --git a/docs/dev-guide/adding-grid-support/adding-grid-support-step-by-step-guide/add-grid-config.md b/docs/dev-guide/adding-grid-support/adding-grid-support-step-by-step-guide/add-grid-config.md index f46a84afa767..bc45fadc9642 100644 --- a/docs/dev-guide/adding-grid-support/adding-grid-support-step-by-step-guide/add-grid-config.md +++ b/docs/dev-guide/adding-grid-support/adding-grid-support-step-by-step-guide/add-grid-config.md @@ -1,3 +1,207 @@ # Add New Grid Configuration to E3SM +In addition to generating input data to support a new grid, several modifications to XML files are required before E3SM can run with the grid. +However, the specific changes will depend on how the grid will be used. The intended model configuration for the new grid will change which files need to be modified. For instance, a grid intended for aquaplanet experiments does not require as many changes as a historical AMIP-style run. + +The guidelines here are meant to outline various possible changes the user should consider when adding support for a new grid for the land and/or atmosphere. +This document cannot be exhaustive, and it is important that the user understands the changes they are making. It is often useful to use a pre-existing grid configuration as a template. +Note that the guidelines here are only relevant for "horizontal" grids in the atmosphere and/or land. +Additional considerations are needed to support a new vertical grid in the atmosphere, which is a topic not currently covered here. + +When setting up a new grid for the atmosphere and/or land model, you will need to edit some or all of these files: + +- `cime_config/config_grids.xml` +- `components/eam/bld/config_files/horiz_grid.xml` +- `components/eam/bld/namelist_files/namelist_defaults_eam.xml` +- `components/eam/bld/namelist_files/namelist_definition.xml` +- `components/elm/bld/namelist_files/namelist_definition.xml` +- `components/elm/bld/namelist_files/namelist_defaults.xml` + +## Mono-Grid vs Bi-Grid vs Tri-Grid + +The mono-bi-tri grid options in E3SM can be confusing, but it is important to understand what these terms mean when adding a new grid to E3SM. At the surface these terms mean that the whole model is either using a single grid for all componennt models, or a combination of 2 or 3 grids shared among the component models. Note that mono-grid and bi-grid terms often ignore that the river model needs to be on its own regular lat-lon grid. + +In practice, "bi" and "tri" grids are most commonly used and the main difference between them comes down to whether the land surface model shares a grid with the atmosphere or not. The component coupler is responsible for facilitating communication between component models, primarily through fluxes, and so mapping files are needed to support a combination of different grids. E3SMv3 uses a tri-grid configuration for production simulations. + +## Grid Naming Conventions + +### Atmosphere + +The atmosphere grid name should always indicate the base "ne" value and whether the physgrid is being used, usually by adding ".pg2" at the end. For a regionally refined mesh (RRM) the grid name should always start with `ne0` followed a descriptive string that includes the region being refined and the degree of refinement. + +**Example**: `ne0np4_northamerica_30x4v1.pg2` + +Note that this example differs from how the North American grid is currently named as `ne0np4_northamericax4v1.pg2`, which indicates a `4x` refinement, but does not indicate the base resolution, which is useful to know. The more informative grid name `ne0np4_northamerica_30x4v1.pg2` makes it clear that unrefined regions are consistent with `ne30pg2`. + +### River (or Land in tri-grid) + +For a rectilinear lat-lon grid used by the land and/or river models the grid name should start with "r" and typically use spacing less than one degree, so they indicate the nominal grid spacing, starting with "0" and omitting the decimal. + +**Examples**: `r05` is 0.5 degree spacing and `r0125` is 1/8 or 0.125 degree spacing. + +### Grid Aliases + +Grid aliases are short strings used to represent the complete set of grids used in the model configuration. +For a mono-grid the convention is that the grid alias is the base mesh written twice to indicate that both atmosphere/land and ocean/sea-ice models are on the same grid. A mono-grid is typically only used for idealized simulations such as aqua planet and RCE, but can also be used for F-compsets if the CICE sea-ice model is used in place of the MPAS sea-ice model (MPASSI). + +**Example**: `ne30pg2_ne30pg2` + +Bi-grid options should indicate two different grids used for atmosphere/land and ocean/sea-ice models. + +**Example**: `ne30pg2_IcoswISC30E3r5` + +Tri-grid options should indicate three different grids used for atmosphere, land, and ocean/sea-ice models, with the land grid appearing in the middle. + +**Example**: `ne30pg2_r05_IcoswISC30E3r5` + +### Grid longnames + +For any combination of grids, the full grid definition has a long form representation that spells out the grid in more detail. + +**Example**: + +```shell + alias: ne4pg2_ne4pg2 + + longname: a%ne4np4.pg2_l%ne4np4.pg2_oi%ne4np4.pg2_r%r05_g%null_w%null_z%null_m%oQU240 + non-default grids are: atm:ne4np4.pg2 lnd:ne4np4.pg2 ocnice:ne4np4.pg2 rof:r05 glc:null wav:null + mask is: oQU240 +``` + +## Defining a New Atmosphere Grid for EAM + +When defining a new atmosphere grid, information needs to be provided on how the grid is constructed. + +To define a new atmosphere grid a line must be added to `components/eam/bld/config_files/horiz_grid.xml` that indicates the number of elements and physics columns. In the lines below for `ne30np4` (without the physgrid) and `ne30pg2` (with the physgrid) you can see the value of `ne` is the same (number of elements along a cube edge), but the number of physics columns is different. + +```xml + + +``` + +An explanation of how to calculate the number of physics columns can be found here: [Atmosphere Grid Overview](../../../EAM/tech-guide/atmosphere-grid-overview.md). + +For a grid with regional refinement, follow the conventions of other grids in this file. There is no formula to calculate the number of columns for RRM grids, but the value can be obtained from the grid files used for mapping. + +```xml + +``` + +## Defining a New Land Grid for ELM + +If you are creating a new grid that will be used by the land model the grid name needs to be added to the list `valid_values` associated with the `res` entry in the file `components/elm/bld/namelist_files/namelist_definition.xml` that holds the definition of namelist variables used by the land model. + +```xml + +Horizontal resolutions +Note: 0.1x0.1, 0.5x0.5, 5x5min, 10x10min, 3x3min and 0.33x0.33 are only used for ELM tools + +``` + +Simply add the name of your new grid to the list of `valid_values`. + +## Using New Grids in Default Namelists + +Each new grid will likely need various new default parameter values to be specified. These parameters can be set for individual simulations by editing the `user_nl_*` files in the case directory, but for these to become defaults any time the grid is used then new defaults need to be specified. + +The lists below show namelist parameters that may need to be specified for a new grid. The creator of a new grid is responsible for understanding these parameters and deciding when new defaults are appropriate. + +### Atmosphere Namelist Parameters + +- `drydep_srf_file` - Data file for surface aerosol deposition +- `bnd_topo` - Surface topography (smoothed for target grid) +- `mesh_file` - HOMME np4 mesh file (exodus format) +- `se_tstep` - HOMME time step [seconds] +- `dt_remap_factor` - HOMME vertical remap factor +- `dt_tracer_factor` - HOMME tracer advection factor +- `hypervis_subcycle_q` - HOMME tracer hyperviscosity factor + +### Land Namelist Parameters + +- `fsurdat` - Surface data file +- `finidat` - Land model initial condition file +- `flanduse_timeseries` - Time-evolving land-use data file + +## Defining a new grid for CIME + +The CIME Case Control system will configure a case according to the component set and grid alias you specify with the `--res` argument. +As part of that configuration, CIME needs to know +how to translate the grid alias and set the paths for domain and mapping files used by the grid so the model can find them at runtime. + +### Adding a New Grid Alias + +Grid aliases are defined in `cime_config/config_grids.xml`. Below is an example grid alias for the `ne30pg2_r05_IcoswISC30E3r5` grid used in E3SMv3 production simulations. + +```xml + + ne30np4.pg2 + r05 + IcoswISC30E3r5 + r05 + null + null + IcoswISC30E3r5 + +``` + +Add a similar block for your new grid. Aliases must be unique within `config_grids.xml` + +### Domain Files + +Domain files are needed for each grid and are specified in the `` section of `cime_config/config_grids.xml`. The default domain files are grouped by the atmosphere grid. The section for the typical `ne30pg2` grid looks as follows: + +```xml + + 21600 + 1 + ... + $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_IcoswISC30E3r5.231121.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_IcoswISC30E3r5.231121.nc + ... + ne30np4.pg2 is Spectral Elem 1-deg grid w/ 2x2 FV physics grid per element: + +``` + +Notice the ellipses `...` are used here to omit all entries that are not relevant to the `ne30pg2_r05_IcoswISC30E3r5` grid. Also, note that all of these paths are relative to the input data path set as `DIN_LOC_ROOT` which has a default for each machine. See [Generating Domain Files](../../../generate_domain_files/index.md) for information about creating domain files. + +### Coupler Mapping Files + +The mapping files used by the component coupler to communicate fluxes between the component models must be specified in the `` section of `cime_config/config_grids.xml`. These are organized for specific pairs of grids, such that tri-grids will require multiple sections. The entries relevant for `ne30pg2_r05_IcoswISC30E3r5` are shown below. + +```xml + + cpl/gridmaps/ne30pg2/map_ne30pg2_to_IcoswISC30E3r5_traave.20231121.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_IcoswISC30E3r5_trbilin.20231121.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_IcoswISC30E3r5-nomask_trbilin.20231121.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_ne30pg2_traave.20231121.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_ne30pg2_traave.20231121.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_IcoswISC30E3r5_trfvnp2.20231121.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_IcoswISC30E3r5_trfvnp2.20231121.nc + +``` + +```xml + + cpl/gridmaps/ne30pg2/map_ne30pg2_to_r05_traave.20231130.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_r05_trfvnp2.230516.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_r05_trbilin.20231130.nc + cpl/gridmaps/ne30pg2/map_r05_to_ne30pg2_traave.20231130.nc + cpl/gridmaps/ne30pg2/map_r05_to_ne30pg2_traave.20231130.nc + +``` + +```xml + + cpl/gridmaps/ne30pg2/map_ne30pg2_to_r05_traave.20231130.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_r05_trfvnp2.230516.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_r05_trbilin.20231130.nc + +``` + +Note that all of these paths are relative to the input data path set as `DIN_LOC_ROOT` which has a default for each machine. Mapping files can be created with +the [ncremap](https://acme-climate.atlassian.net/wiki/spaces/DOC/pages/754286611/Regridding+E3SM+Data+with+ncremap) utility in NCO + Back to step-by-step guide for [Adding Support for New Grids](../adding-grid-support-step-by-step-guide.md) diff --git a/docs/index.md b/docs/index.md index 4d979cf0afee..6a30bcf55451 100644 --- a/docs/index.md +++ b/docs/index.md @@ -24,6 +24,7 @@ research problems and Department of Energy mission needs while efficiently using - [MPAS-Ocean](./MPAS-Ocean/index.md) - [MPAS-seaice](./MPAS-seaice/index.md) - [Omega](https://docs.e3sm.org/Omega/omega/) — not yet supported. +- [Data Models](./Data-Models/index.md) ## Tools diff --git a/docs/refs/eam.bib b/docs/refs/eam.bib index ff4415a519b8..84bdd2499ce4 100644 --- a/docs/refs/eam.bib +++ b/docs/refs/eam.bib @@ -1035,3 +1035,128 @@ @article{neale_description_2012 journal = {UNKNOWN}, year = {2012}, } + +@article{xie_an_2020, + title = {An Orographic-Drag Parametrization Scheme Including Orographic Anisotropy for All Flow Directions}, + url = {https://agupubs.onlinelibrary.wiley.com/doi/full/10.1029/2019MS001921/}, + doi = {10.1029/2019MS001921}, + language = {en}, + urldate = {2024-11-19}, + author = {J., Xie and M.,Zhang and Z., Xie and H., Liu and Z., Chai and J., He and H. Zhang}, + journal = {Journal of Advances in Modeling Earth Systems}, + year = {2020}, +} + +@article{beljaars_a_2020, + title = {A new parametrization of turbulent orographic form drag}, + url = {https://rmets.onlinelibrary.wiley.com/doi/abs/10.1256/qj.03.73/}, + doi = {10.1256/qj.03.73}, + language = {en}, + urldate = {2024-11-19}, + author = {A. Beljaars, A. Brown, N. Wood}, + journal = {Quarterly Journal of the Royal Meterological Society}, + year = {2004}, +} + +@article{tsiringakis_small_2020, + title = {Small-scale orographic gravity wave drag in stable boundary layers and its impact on synoptic systems and near-surface meteorology}, + url = {https://rmets.onlinelibrary.wiley.com/doi/abs/10.1002/qj.3021}, + doi = {10.1002/qj.3021}, + language = {en}, + urldate = {2024-11-19}, + author = {A. Tsiringakis, G.J. Steeneveld, A.A.M. Holtslag}, + journal = {Quarterly Journal of the Royal Meterological Society}, + year = {2017}, +} + +@article{mcfarlane_the_1987, + title = {The Effect of Orographically Excited Gravity Wave Drag on the General Circulation of the Lower Stratosphere and Troposphere}, + url = {https://journals.ametsoc.org/view/journals/atsc/44/14/1520-0469_1987_044_1775_teooeg_2_0_co_2.xml}, + doi = {10.1175/1520-0469(1987)044<1775:TEOOEG>2.0.CO;2}, + language = {en}, + urldate = {2024-11-19}, + author = {N.A. McFarlane}, + journal = {Journal of the Atmospheric Sciences}, + year = {1987}, +} + +@article{richter_the_2010, + title = {The Effect of Orographically Excited Gravity Wave Drag on the General Circulation of the Lower Stratosphere and Troposphere}, + url = {https://journals.ametsoc.org/view/journals/atsc/67/1/2009jas3112.1.xml?tab_body=pdf}, + doi = {10.1175/2009JAS3112.1}, + language = {en}, + urldate = {2024-11-19}, + author = {J.H. Richter, F. Sassi, R.R. Garcia}, + journal = {Journal of the Atmospheric Sciences}, + year = {2010}, +} + +@article{holtslag_preface_2006, + title = {Preface:GEWEXatmospheric boundary-layer study (GABLS) on stable boundary layers}, + url = {https://link.springer.com/article/10.1007/s10546-005-9008-6}, + doi = {10.1007/s10546-005-9008-6}, + language = {en}, + urldate = {2024-11-19}, + author = {A.A Holtslag}, + journal = {Boundary-Layer Meterology}, + year = {2006}, +} + +@article{blackburn_APE_context_2013, + title = {Context and {Aims} of the {Aqua}-{Planet} {Experiment}}, + volume = {91A}, + issn = {2186-9057}, + doi = {10.2151/jmsj.2013-A01}, + journal = {Journal of the Meteorological Society of Japan. Ser. II}, + author = {BLACKBURN, Michael and HOSKINS, Brian J.}, + month = oct, + year = {2013}, + keywords = {precipitation, circulation models, comparison of atmospheric general, gcms, intertropical convergence zone, itcz, modelling hierarchy, tropical circulation}, + pages = {1--15}, +} + +@article{wing_rcemip1_2018, + title = {Radiative–convective equilibrium model intercomparison project}, + volume = {11}, + issn = {1991-959X}, + doi = {10.5194/gmd-11-793-2018}, + language = {English}, + number = {2}, + journal = {Geoscientific Model Development}, + author = {Wing, Allison A. and Reed, Kevin A. and Satoh, Masaki and Stevens, Bjorn and Bony, Sandrine and Ohno, Tomoki}, + month = mar, + year = {2018}, + note = {Publisher: Copernicus GmbH}, + pages = {793--813}, + file = {Full Text PDF:/Users/hannah6/Zotero/storage/KHKQVU33/Wing et al. - 2018 - Radiative–convective equilibrium model intercomparison project.pdf:application/pdf}, +} + +@article{wing_rcemip2_2024, + title = {{RCEMIP}-{II}: mock-{Walker} simulations as phase {II} of the radiative–convective equilibrium model intercomparison project}, + volume = {17}, + issn = {1991-959X}, + shorttitle = {{RCEMIP}-{II}}, + doi = {10.5194/gmd-17-6195-2024}, + language = {English}, + number = {16}, + journal = {Geoscientific Model Development}, + author = {Wing, Allison A. and Silvers, Levi G. and Reed, Kevin A.}, + month = aug, + year = {2024}, + note = {Publisher: Copernicus GmbH}, + pages = {6195--6225}, + file = {Full Text PDF:/Users/hannah6/Zotero/storage/IY2SP66J/Wing et al. - 2024 - RCEMIP-II mock-Walker simulations as phase II of the radiative–convective equilibrium model interco.pdf:application/pdf}, +} + +@article { Zarzycki_TC-ocn-cpl_2016, + author = "Colin M. Zarzycki", + title = "Tropical Cyclone Intensity Errors Associated with Lack of Two-Way Ocean Coupling in High-Resolution Global Simulations", + journal = "Journal of Climate", + year = "2016", + publisher = "American Meteorological Society", + address = "Boston MA, USA", + volume = "29", + number = "23", + doi = "10.1175/JCLI-D-16-0273.1", + pages = "8589 - 8610", +} diff --git a/driver-mct/cime_config/buildnml b/driver-mct/cime_config/buildnml index 4938e9da0b18..c65a6047555c 100755 --- a/driver-mct/cime_config/buildnml +++ b/driver-mct/cime_config/buildnml @@ -41,6 +41,7 @@ def _create_drv_namelists(case, infile, confdir, nmlgen, files): config['CPL_EPBAL'] = case.get_value('CPL_EPBAL') config['FLDS_WISO'] = case.get_value('FLDS_WISO') config['FLDS_POLAR'] = case.get_value('FLDS_POLAR') + config['FLDS_TF'] = case.get_value('FLDS_TF') config['BUDGETS'] = case.get_value('BUDGETS') config['MACH'] = case.get_value('MACH') config['MPILIB'] = case.get_value('MPILIB') diff --git a/driver-mct/cime_config/config_component.xml b/driver-mct/cime_config/config_component.xml index 8cddad0abf89..666474353c79 100644 --- a/driver-mct/cime_config/config_component.xml +++ b/driver-mct/cime_config/config_component.xml @@ -1898,40 +1898,58 @@ glc2ocn runoff mapping file decomp type for ice runoff - + char idmap_ignore run_domain env_run.xml - ocn2glc flux mapping file - the default value idmap_ignore, if set, will be ignored by buildnml and + ocn2glc shelf flux mapping file - the default value idmap_ignore, if set, will be ignored by buildnml and will generate a runtime error if in fact a file is required for the given compset - + char X,Y Y run_domain env_run.xml - ocn2glc flux mapping file decomp type + ocn2glc shelf flux mapping file decomp type - + char idmap_ignore run_domain env_run.xml - ocn2glc state mapping file - the default value idmap_ignore, if set, will be ignored by buildnml and + ocn2glc shelf state mapping file - the default value idmap_ignore, if set, will be ignored by buildnml and will generate a runtime error if in fact a file is required for the given compset - + char X,Y Y run_domain env_run.xml - ocn2glc state mapping file decomp type + ocn2glc shelf state mapping file decomp type + + + + char + idmap_ignore + run_domain + env_run.xml + ocn2glc state mapping file for thermal forcing - the default value idmap_ignore, if set, will be ignored by buildnml and + will generate a runtime error if in fact a file is required for the given compset + + + + char + X,Y + Y + run_domain + env_run.xml + ocn2glc thermal forcing state mapping file decomp type @@ -2628,7 +2646,7 @@ char - netcdf,pnetcdf,netcdf4p,netcdf4c,adios,hdf5,default + netcdf,pnetcdf,netcdf4p,netcdf4c,adios,adiosc,hdf5,default run_pio env_run.xml pio io type diff --git a/driver-mct/cime_config/config_component_e3sm.xml b/driver-mct/cime_config/config_component_e3sm.xml index 8fc93b607d46..c16a493f0453 100755 --- a/driver-mct/cime_config/config_component_e3sm.xml +++ b/driver-mct/cime_config/config_component_e3sm.xml @@ -185,6 +185,18 @@ Turn on the passing of polar fields through the coupler + + logical + TRUE,FALSE + FALSE + + TRUE + + run_flags + env_run.xml + Turn on the passing of ocean thermal forcing fields through the coupler + + char minus1p8,linear_salt,mushy @@ -412,6 +424,7 @@ 144 432 864 + 864 144 96 48 diff --git a/driver-mct/cime_config/namelist_definition_drv.xml b/driver-mct/cime_config/namelist_definition_drv.xml index 982acabe64d8..a0cfc87e5266 100644 --- a/driver-mct/cime_config/namelist_definition_drv.xml +++ b/driver-mct/cime_config/namelist_definition_drv.xml @@ -149,6 +149,18 @@ + + logical + seq_flds + seq_cplflds_inparm + + If set to .true. thermal forcing fields will be passed from the ocean to the coupler. + + + $FLDS_TF + + + logical seq_flds @@ -4531,20 +4543,50 @@ - + + char + mapping + abs + seq_maps + + ocn to glc shelf mapping file for fluxes + + + $OCN2GLC_SHELF_FMAPNAME + + + + + char + mapping + seq_maps + + The type of mapping desired, either "source" or "destination" mapping. + X is associated with rearrangement of the source grid to the + destination grid and then local mapping. Y is associated with mapping + on the source grid and then rearrangement and sum to the destination + grid. + + + $OCN2GLC_SHELF_FMAPTYPE + X + + + + char mapping abs seq_maps - ocn to glc flux mapping file for fluxes + ocn to glc shelf mapping file for states - $OCN2GLC_FMAPNAME + $OCN2GLC_SHELF_SMAPNAME - + char mapping seq_maps @@ -4556,25 +4598,25 @@ grid.
- $OCN2GLC_FMAPTYPE + $OCN2GLC_SHELF_SMAPTYPE X - + char mapping abs seq_maps - ocn to glc state mapping file for states + ocn to glc state mapping file for thermal forcing state fields - $OCN2GLC_SMAPNAME + $OCN2GLC_TF_SMAPNAME - + char mapping seq_maps @@ -4586,7 +4628,7 @@ grid. - $OCN2GLC_SMAPTYPE + $OCN2GLC_TF_SMAPTYPE X diff --git a/driver-mct/main/cime_comp_mod.F90 b/driver-mct/main/cime_comp_mod.F90 index 7cf442b9413d..4614474bac3e 100644 --- a/driver-mct/main/cime_comp_mod.F90 +++ b/driver-mct/main/cime_comp_mod.F90 @@ -145,7 +145,8 @@ module cime_comp_mod ! diagnostic routines use seq_diag_mct, only : seq_diag_zero_mct , seq_diag_avect_mct, seq_diag_lnd_mct use seq_diag_mct, only : seq_diag_rof_mct , seq_diag_ocn_mct , seq_diag_atm_mct - use seq_diag_mct, only : seq_diag_ice_mct , seq_diag_accum_mct, seq_diag_print_mct + use seq_diag_mct, only : seq_diag_ice_mct , seq_diag_glc_mct + use seq_diag_mct, only : seq_diag_accum_mct, seq_diag_print_mct use seq_diagBGC_mct, only : seq_diagBGC_zero_mct , seq_diagBGC_avect_mct, seq_diagBGC_lnd_mct use seq_diagBGC_mct, only : seq_diagBGC_rof_mct , seq_diagBGC_ocn_mct , seq_diagBGC_atm_mct use seq_diagBGC_mct, only : seq_diagBGC_ice_mct , seq_diagBGC_accum_mct @@ -434,6 +435,7 @@ module cime_comp_mod logical :: lnd_c2_glc ! .true. => lnd to glc coupling on logical :: ocn_c2_atm ! .true. => ocn to atm coupling on logical :: ocn_c2_ice ! .true. => ocn to ice coupling on + logical :: ocn_c2_glctf ! .true. => ocn to glc thermal forcing coupling on logical :: ocn_c2_glcshelf ! .true. => ocn to glc ice shelf coupling on logical :: ocn_c2_wav ! .true. => ocn to wav coupling on logical :: ocn_c2_rof ! .true. => ocn to rof coupling on @@ -1674,6 +1676,7 @@ subroutine cime_init() ocn_prognostic=ocn_prognostic, & ocnrof_prognostic=ocnrof_prognostic, & ocn_c2_glcshelf=ocn_c2_glcshelf, & + ocn_c2_glctf=ocn_c2_glctf, & glc_prognostic=glc_prognostic, & rof_prognostic=rof_prognostic, & rofocn_prognostic=rofocn_prognostic, & @@ -1873,8 +1876,9 @@ subroutine cime_init() write(logunit,F0L)'lnd_c2_rof = ',lnd_c2_rof write(logunit,F0L)'lnd_c2_glc = ',lnd_c2_glc write(logunit,F0L)'ocn_c2_atm = ',ocn_c2_atm - write(logunit,F0L)'ocn_c2_ice = ',ocn_c2_ice write(logunit,F0L)'ocn_c2_glcshelf = ',ocn_c2_glcshelf + write(logunit,F0L)'ocn_c2_glctf = ',ocn_c2_glctf + write(logunit,F0L)'ocn_c2_ice = ',ocn_c2_ice write(logunit,F0L)'ocn_c2_wav = ',ocn_c2_wav write(logunit,F0L)'ocn_c2_rof = ',ocn_c2_rof write(logunit,F0L)'ice_c2_atm = ',ice_c2_atm @@ -1959,7 +1963,7 @@ subroutine cime_init() endif if ((ocn_c2_glcshelf .and. .not. glcshelf_c2_ocn) .or. (glcshelf_c2_ocn .and. .not. ocn_c2_glcshelf)) then ! Current logic will not allow this to be true, but future changes could make it so, which may be nonsensical - call shr_sys_abort(subname//' ERROR: if glc_c2_ocn must also have ocn_c2_glc and vice versa. '//& + call shr_sys_abort(subname//' ERROR: if glcshelf_c2_ocn must also have ocn_c2_glcshelf and vice versa. '//& 'Boundary layer fluxes calculated in coupler require input from both components.') endif if (rofice_present .and. .not.rof_present) then @@ -2028,7 +2032,7 @@ subroutine cime_init() call prep_rof_init(infodata, lnd_c2_rof, atm_c2_rof, ocn_c2_rof) - call prep_glc_init(infodata, lnd_c2_glc, ocn_c2_glcshelf) + call prep_glc_init(infodata, lnd_c2_glc, ocn_c2_glctf, ocn_c2_glcshelf) call prep_wav_init(infodata, atm_c2_wav, ocn_c2_wav, ice_c2_wav) @@ -3052,8 +3056,12 @@ subroutine cime_run() !---------------------------------------------------------- !| GLC SETUP-SEND !---------------------------------------------------------- - if (glc_present .and. glcrun_alarm) then - call cime_run_glc_setup_send(lnd2glc_averaged_now, prep_glc_accum_avg_called) + if (glc_present) then + if (glcrun_alarm) then + call cime_run_glc_setup_send(lnd2glc_averaged_now, prep_glc_accum_avg_called) + else + call prep_glc_zero_fields() + endif endif ! ------------------------------------------------------------------------ @@ -3100,6 +3108,7 @@ subroutine cime_run() endif endif + !---------------------------------------------------------- !| Budget with old fractions !---------------------------------------------------------- @@ -4216,12 +4225,16 @@ subroutine cime_run_ocnglc_coupling() if (glc_present) then + ! create o2x_gx for either ocn-glc coupling or ocn-glc shelf coupling + if (ocn_c2_glctf .or. (ocn_c2_glcshelf .and. glcshelf_c2_ocn)) then + call prep_glc_calc_o2x_gx(ocn_c2_glctf, ocn_c2_glcshelf, timer='CPL:glcprep_ocn2glc') !remap ocean fields to o2x_g at ocean couping interval + endif + + ! if ice-shelf coupling is on, now proceed to handle those calculations here in the coupler if (ocn_c2_glcshelf .and. glcshelf_c2_ocn) then ! the boundary flux calculations done in the coupler require inputs from both GLC and OCN, ! so they will only be valid if both OCN->GLC and GLC->OCN - call prep_glc_calc_o2x_gx(timer='CPL:glcprep_ocn2glc') !remap ocean fields to o2x_g at ocean couping interval - call prep_glc_calculate_subshelf_boundary_fluxes ! this is actual boundary layer flux calculation !this outputs !x2g_g/g2x_g, where latter is going @@ -4348,7 +4361,7 @@ subroutine cime_run_glc_setup_send(lnd2glc_averaged_now, prep_glc_accum_avg_call if (drv_threading) call seq_comm_setnthreads(nthreads_CPLID) ! NOTE - only create appropriate input to glc if the avg_alarm is on - if (lnd_c2_glc .or. ocn_c2_glcshelf) then + if (lnd_c2_glc .or. ocn_c2_glctf .or. ocn_c2_glcshelf) then if (glcrun_avg_alarm) then call prep_glc_accum_avg(timer='CPL:glcprep_avg', & lnd2glc_averaged_now=lnd2glc_averaged_now) @@ -4361,6 +4374,13 @@ subroutine cime_run_glc_setup_send(lnd2glc_averaged_now, prep_glc_accum_avg_call call prep_glc_mrg_lnd(infodata, fractions_gx, timer_mrg='CPL:glcprep_mrgx2g') endif + if (ocn_c2_glctf) then + ! note: o2x_gx is handled in prep_glc_calc_o2x_gx, which is called + ! from cime_run_ocnglc_coupling in this module + call prep_glc_mrg_ocn(infodata, fractions_gx, timer_mrg='CPL:glcprep_mrgocnx2g') + endif + + call component_diag(infodata, glc, flow='x2c', comment='send glc', & info_debug=info_debug, timer_diag='CPL:glcprep_diagav') @@ -4749,6 +4769,9 @@ subroutine cime_run_calc_budgets1(in_cplrun) if (ice_present) then call seq_diag_ice_mct(ice(ens1), fractions_ix(ens1), infodata, do_x2i=.true.) endif + if (glc_present) then + call seq_diag_glc_mct(glc(ens1), fractions_gx(ens1), infodata, do_x2g=.true.) + endif if (do_bgc_budgets) then if (rof_present) then call seq_diagBGC_rof_mct(rof(ens1), fractions_rx(ens1), infodata) @@ -4788,6 +4811,9 @@ subroutine cime_run_calc_budgets2(in_cplrun) if (ice_present) then call seq_diag_ice_mct(ice(ens1), fractions_ix(ens1), infodata, do_i2x=.true.) endif + if (glc_present) then + call seq_diag_glc_mct(glc(ens1), fractions_gx(ens1), infodata, do_g2x=.true.) + endif if (do_bgc_budgets) then if (atm_present) then call seq_diagBGC_atm_mct(atm(ens1), fractions_ax(ens1), infodata, do_a2x=.true., do_x2a=.true.) @@ -5595,3 +5621,4 @@ function copy_and_trim_rpointer_file(src, dst) result(out) end function copy_and_trim_rpointer_file end module cime_comp_mod + diff --git a/driver-mct/main/prep_glc_mod.F90 b/driver-mct/main/prep_glc_mod.F90 index 07aeb9890bda..890edc82295e 100644 --- a/driver-mct/main/prep_glc_mod.F90 +++ b/driver-mct/main/prep_glc_mod.F90 @@ -31,6 +31,7 @@ module prep_glc_mod public :: prep_glc_init public :: prep_glc_mrg_lnd + public :: prep_glc_mrg_ocn public :: prep_glc_accum_lnd public :: prep_glc_accum_ocn @@ -45,6 +46,7 @@ module prep_glc_mod public :: prep_glc_get_l2gacc_lx public :: prep_glc_get_l2gacc_lx_one_instance public :: prep_glc_get_l2gacc_lx_cnt + public :: prep_glc_get_l2gacc_lx_cnt_avg public :: prep_glc_get_o2x_gx public :: prep_glc_get_x2gacc_gx @@ -53,8 +55,8 @@ module prep_glc_mod public :: prep_glc_get_mapper_Sl2g public :: prep_glc_get_mapper_Fl2g - public :: prep_glc_get_mapper_So2g - public :: prep_glc_get_mapper_Fo2g + public :: prep_glc_get_mapper_So2g_shelf + public :: prep_glc_get_mapper_Fo2g_shelf public :: prep_glc_calculate_subshelf_boundary_fluxes @@ -76,8 +78,9 @@ module prep_glc_mod ! mappers type(seq_map), pointer :: mapper_Sl2g type(seq_map), pointer :: mapper_Fl2g - type(seq_map), pointer :: mapper_So2g - type(seq_map), pointer :: mapper_Fo2g + type(seq_map), pointer :: mapper_So2g_shelf + type(seq_map), pointer :: mapper_Fo2g_shelf + type(seq_map), pointer :: mapper_So2g_tf type(seq_map), pointer :: mapper_Fg2l ! attribute vectors @@ -91,6 +94,7 @@ module prep_glc_mod type(mct_aVect), pointer :: l2gacc_lx(:) ! Lnd export, lnd grid, cpl pes - allocated in driver integer , target :: l2gacc_lx_cnt ! l2gacc_lx: number of time samples accumulated + integer , target :: l2gacc_lx_cnt_avg ! l2gacc_lx: number of time samples averaged ! other module variables integer :: mpicom_CPLID ! MPI cpl communicator @@ -135,7 +139,7 @@ module prep_glc_mod !================================================================================================ - subroutine prep_glc_init(infodata, lnd_c2_glc, ocn_c2_glcshelf) + subroutine prep_glc_init(infodata, lnd_c2_glc, ocn_c2_glctf, ocn_c2_glcshelf) !--------------------------------------------------------------- ! Description @@ -144,7 +148,8 @@ subroutine prep_glc_init(infodata, lnd_c2_glc, ocn_c2_glcshelf) ! Arguments type (seq_infodata_type) , intent(inout) :: infodata logical , intent(in) :: lnd_c2_glc ! .true. => lnd to glc coupling on - logical , intent(in) :: ocn_c2_glcshelf ! .true. => ocn to glc coupling on + logical , intent(in) :: ocn_c2_glctf ! .true. => ocn to glc thermal forcing coupling on + logical , intent(in) :: ocn_c2_glcshelf ! .true. => ocn to glc shelf coupling on ! ! Local Variables integer :: eli, egi, eoi @@ -178,8 +183,9 @@ subroutine prep_glc_init(infodata, lnd_c2_glc, ocn_c2_glcshelf) allocate(mapper_Sl2g) allocate(mapper_Fl2g) - allocate(mapper_So2g) - allocate(mapper_Fo2g) + allocate(mapper_So2g_shelf) + allocate(mapper_So2g_tf) + allocate(mapper_Fo2g_shelf) allocate(mapper_Fg2l) smb_renormalize = prep_glc_do_renormalize_smb(infodata) @@ -195,6 +201,7 @@ subroutine prep_glc_init(infodata, lnd_c2_glc, ocn_c2_glcshelf) call mct_aVect_zero(l2gacc_lx(eli)) end do l2gacc_lx_cnt = 0 + l2gacc_lx_cnt_avg = 0 end if if (glc_present .and. lnd_c2_glc) then @@ -249,8 +256,8 @@ subroutine prep_glc_init(infodata, lnd_c2_glc, ocn_c2_glcshelf) end if - if (glc_present .and. ocn_c2_glcshelf) then - + ! setup needed for either kind of ocn2glc coupling + if (glc_present .and. (ocn_c2_glctf .or. ocn_c2_glcshelf)) then call seq_comm_getData(CPLID, & mpicom=mpicom_CPLID, iamroot=iamroot_CPLID) @@ -275,21 +282,35 @@ subroutine prep_glc_init(infodata, lnd_c2_glc, ocn_c2_glcshelf) x2gacc_gx_cnt = 0 samegrid_go = .true. if (trim(ocn_gnam) /= trim(glc_gnam)) samegrid_go = .false. + end if + + ! setup needed for ocn2glc TF coupling + if (glc_present .and. ocn_c2_glctf) then if (iamroot_CPLID) then write(logunit,*) ' ' - write(logunit,F00) 'Initializing mapper_So2g' + write(logunit,F00) 'Initializing mapper_So2g_tf' end if - call seq_map_init_rcfile(mapper_So2g, ocn(1), glc(1), & - 'seq_maps.rc','ocn2glc_smapname:','ocn2glc_smaptype:',samegrid_go, & - 'mapper_So2g initialization',esmf_map_flag) + call seq_map_init_rcfile(mapper_So2g_tf, ocn(1), glc(1), & + 'seq_maps.rc','ocn2glc_tf_smapname:','ocn2glc_tf_smaptype:',samegrid_go, & + 'mapper_So2g_tf initialization',esmf_map_flag) + end if + + ! setup needed for ocn2glcshelf coupling + if (glc_present .and. ocn_c2_glcshelf) then if (iamroot_CPLID) then write(logunit,*) ' ' - write(logunit,F00) 'Initializing mapper_Fo2g' + write(logunit,F00) 'Initializing mapper_So2g_shelf' end if - call seq_map_init_rcfile(mapper_Fo2g, ocn(1), glc(1), & - 'seq_maps.rc','ocn2glc_fmapname:','ocn2glc_fmaptype:',samegrid_go, & - 'mapper_Fo2g initialization',esmf_map_flag) - + call seq_map_init_rcfile(mapper_So2g_shelf, ocn(1), glc(1), & + 'seq_maps.rc','ocn2glc_shelf_smapname:','ocn2glc_shelf_smaptype:',samegrid_go, & + 'mapper_So2g_shelf initialization',esmf_map_flag) + if (iamroot_CPLID) then + write(logunit,*) ' ' + write(logunit,F00) 'Initializing mapper_Fo2g_shelf' + end if + call seq_map_init_rcfile(mapper_Fo2g_shelf, ocn(1), glc(1), & + 'seq_maps.rc','ocn2glc_shelf_fmapname:','ocn2glc_shelf_fmaptype:',samegrid_go, & + 'mapper_Fo2g_shelf initialization',esmf_map_flag) !Initialize module-level arrays associated with compute_melt_fluxes allocate(oceanTemperature(lsize_g)) allocate(oceanSalinity(lsize_g)) @@ -307,10 +328,9 @@ subroutine prep_glc_init(infodata, lnd_c2_glc, ocn_c2_glcshelf) ! TODO: Can we allocate these only while used or are we worried about performance hit? ! TODO: add deallocates! - call shr_sys_flush(logunit) - end if + call shr_sys_flush(logunit) end subroutine prep_glc_init @@ -502,6 +522,7 @@ subroutine prep_glc_accum_avg(timer, lnd2glc_averaged_now) call mct_avect_avg(l2gacc_lx(eli), l2gacc_lx_cnt) end do end if + l2gacc_lx_cnt_avg = l2gacc_lx_cnt l2gacc_lx_cnt = 0 ! Accumulation for OCN @@ -521,6 +542,154 @@ subroutine prep_glc_accum_avg(timer, lnd2glc_averaged_now) end subroutine prep_glc_accum_avg + !================================================================================================ + + subroutine prep_glc_mrg_ocn(infodata, fractions_gx, timer_mrg) + + !--------------------------------------------------------------- + ! Description + ! Merge glc inputs + ! + ! Arguments + type(seq_infodata_type) , intent(in) :: infodata + type(mct_aVect) , intent(in) :: fractions_gx(:) + character(len=*) , intent(in) :: timer_mrg + ! + ! Local Variables + integer :: egi, eoi, efi + type(mct_avect), pointer :: x2g_gx + character(*), parameter :: subname = '(prep_glc_mrg_ocn)' + !--------------------------------------------------------------- + + call t_drvstartf (trim(timer_mrg),barrier=mpicom_CPLID) + do egi = 1,num_inst_glc + ! Use fortran mod to address ensembles in merge + eoi = mod((egi-1),num_inst_ocn) + 1 + efi = mod((egi-1),num_inst_frc) + 1 + + x2g_gx => component_get_x2c_cx(glc(egi)) + call prep_glc_merge_ocn_forcing(o2x_gx(eoi), fractions_gx(efi), x2g_gx) + enddo + call t_drvstopf (trim(timer_mrg)) + + end subroutine prep_glc_mrg_ocn + + !================================================================================================ + + subroutine prep_glc_merge_ocn_forcing( o2x_g, fractions_g, x2g_g ) + + !----------------------------------------------------------------------- + ! Description + ! "Merge" ocean forcing for glc input. + ! + ! State fields are copied directly, meaning that averages are taken just over the + ! ocean-covered portion of the glc domain. + ! + ! Flux fields are downweighted by landfrac, which effectively sends a 0 flux from the + ! non-ocean-covered portion of the glc domain. + ! + ! Arguments + type(mct_aVect), intent(inout) :: o2x_g ! input + type(mct_aVect), intent(in) :: fractions_g + type(mct_aVect), intent(inout) :: x2g_g ! output + !----------------------------------------------------------------------- + + integer :: num_flux_fields + integer :: num_state_fields + integer :: nflds + integer :: i,n + integer :: mrgstr_index + integer :: index_o2x + integer :: index_x2g + integer :: index_ofrac + integer :: lsize + logical :: iamroot + logical, save :: first_time = .true. + character(CL),allocatable :: mrgstr(:) ! temporary string + character(CL) :: field ! string converted to char + character(*), parameter :: subname = '(prep_glc_merge_ocn_forcing) ' + + !----------------------------------------------------------------------- + + call seq_comm_getdata(CPLID, iamroot=iamroot) + lsize = mct_aVect_lsize(x2g_g) + + !num_flux_fields = shr_string_listGetNum(trim(seq_flds_x2g_fluxes_from_ocn)) + num_flux_fields = 0 + num_state_fields = shr_string_listGetNum(trim(seq_flds_x2g_tf_states_from_ocn)) + + if (first_time) then + nflds = num_flux_fields + num_state_fields + allocate(mrgstr(nflds)) + end if + + mrgstr_index = 1 + + do i = 1, num_state_fields + call seq_flds_getField(field, i, seq_flds_x2g_tf_states_from_ocn) + index_o2x = mct_aVect_indexRA(o2x_g, trim(field)) + index_x2g = mct_aVect_indexRA(x2g_g, trim(field)) + + if (first_time) then + mrgstr(mrgstr_index) = subname//'x2g%'//trim(field)//' =' // & + ' = o2x%'//trim(field) + end if + + do n = 1, lsize + x2g_g%rAttr(index_x2g,n) = o2x_g%rAttr(index_o2x,n) + end do + + mrgstr_index = mrgstr_index + 1 + enddo + + !index_lfrac = mct_aVect_indexRA(fractions_g,"lfrac") + !do i = 1, num_flux_fields + + ! call seq_flds_getField(field, i, seq_flds_x2g_fluxes_from_lnd) + ! index_l2x = mct_aVect_indexRA(l2x_g, trim(field)) + ! index_x2g = mct_aVect_indexRA(x2g_g, trim(field)) + + ! if (trim(field) == qice_fieldname) then + + ! if (first_time) then + ! mrgstr(mrgstr_index) = subname//'x2g%'//trim(field)//' =' // & + ! ' = l2x%'//trim(field) + ! end if + + ! ! treat qice as if it were a state variable, with a simple copy. + ! do n = 1, lsize + ! x2g_g%rAttr(index_x2g,n) = l2x_g%rAttr(index_l2x,n) + ! end do + + ! else + ! write(logunit,*) subname,' ERROR: Flux fields other than ', & + ! qice_fieldname, ' currently are not handled in lnd2glc remapping.' + ! write(logunit,*) '(Attempt to handle flux field <', trim(field), '>.)' + ! write(logunit,*) 'Substantial thought is needed to determine how to remap other fluxes' + ! write(logunit,*) 'in a smooth, conservative manner.' + ! call shr_sys_abort(subname//& + ! ' ERROR: Flux fields other than qice currently are not handled in lnd2glc remapping.') + ! endif ! qice_fieldname + + ! mrgstr_index = mrgstr_index + 1 + + !end do + + if (first_time) then + if (iamroot) then + write(logunit,'(A)') subname//' Summary:' + do i = 1,nflds + write(logunit,'(A)') trim(mrgstr(i)) + enddo + endif + deallocate(mrgstr) + endif + + first_time = .false. + + end subroutine prep_glc_merge_ocn_forcing + + !================================================================================================ subroutine prep_glc_mrg_lnd(infodata, fractions_gx, timer_mrg) @@ -604,7 +773,7 @@ subroutine prep_glc_merge_lnd_forcing( l2x_g, fractions_g, x2g_g ) mrgstr_index = 1 do i = 1, num_state_fields - call seq_flds_getField(field, i, seq_flds_x2g_states) + call seq_flds_getField(field, i, seq_flds_x2g_states_from_lnd) index_l2x = mct_aVect_indexRA(l2x_g, trim(field)) index_x2g = mct_aVect_indexRA(x2g_g, trim(field)) @@ -668,13 +837,15 @@ subroutine prep_glc_merge_lnd_forcing( l2x_g, fractions_g, x2g_g ) end subroutine prep_glc_merge_lnd_forcing - subroutine prep_glc_calc_o2x_gx(timer) + subroutine prep_glc_calc_o2x_gx(ocn_c2_glctf, ocn_c2_glcshelf, timer) !--------------------------------------------------------------- ! Description ! Create o2x_gx ! Arguments character(len=*), intent(in) :: timer + logical, intent(in) :: ocn_c2_glctf + logical, intent(in) :: ocn_c2_glcshelf character(*), parameter :: subname = '(prep_glc_calc_o2x_gx)' ! Local Variables @@ -684,8 +855,14 @@ subroutine prep_glc_calc_o2x_gx(timer) call t_drvstartf (trim(timer),barrier=mpicom_CPLID) do eoi = 1,num_inst_ocn o2x_ox => component_get_c2x_cx(ocn(eoi)) - call seq_map_map(mapper_So2g, o2x_ox, o2x_gx(eoi), & - fldlist=seq_flds_x2g_states_from_ocn,norm=.true.) + if (ocn_c2_glctf) then + call seq_map_map(mapper_So2g_tf, o2x_ox, o2x_gx(eoi), & + fldlist=seq_flds_x2g_tf_states_from_ocn,norm=.true.) + end if + if (ocn_c2_glcshelf) then + call seq_map_map(mapper_So2g_shelf, o2x_ox, o2x_gx(eoi), & + fldlist=seq_flds_x2g_shelf_states_from_ocn,norm=.true.) + end if enddo call t_drvstopf (trim(timer)) @@ -850,8 +1027,8 @@ subroutine prep_glc_calculate_subshelf_boundary_fluxes !Done here instead of in glc-frequency mapping so it happens within ocean coupling interval. ! Also could map o2x_ox->o2x_gx(1) but using x2g_gx as destination allows us to see ! these fields on the GLC grid of the coupler history file, which helps with debugging. - call seq_map_map(mapper_So2g, o2x_ox, x2g_gx, & - fldlist=seq_flds_x2g_states_from_ocn,norm=.true.) + call seq_map_map(mapper_So2g_shelf, o2x_ox, x2g_gx, & + fldlist=seq_flds_x2g_shelf_states_from_ocn,norm=.true.) ! inputs to melt flux calculation index_x2g_So_blt = mct_avect_indexra(x2g_gx,'So_blt',perrwith='quiet') @@ -950,6 +1127,7 @@ subroutine prep_glc_zero_fields() type(mct_avect), pointer :: x2g_gx !--------------------------------------------------------------- + do egi = 1,num_inst_glc x2g_gx => component_get_x2c_cx(glc(egi)) call mct_aVect_zero(x2g_gx) @@ -1195,8 +1373,9 @@ subroutine prep_glc_renormalize_smb(eli, fractions_lx, g2x_gx, mapper_Fg2l, area aream_l(:) = dom_l%data%rAttr(km,:) ! Export land fractions from fractions_lx to a local array + ! Note that for E3SM we are using lfrin instead of lfrac allocate(lfrac(lsize_l)) - call mct_aVect_exportRattr(fractions_lx, "lfrac", lfrac) + call mct_aVect_exportRattr(fractions_lx, "lfrin", lfrac) ! Map Sg_icemask from the glc grid to the land grid. ! This may not be necessary, if Sg_icemask_l has already been mapped from Sg_icemask_g. @@ -1379,6 +1558,8 @@ subroutine prep_glc_renormalize_smb(eli, fractions_lx, g2x_gx, mapper_Fg2l, area endif if (iamroot) then + write(logunit,*) 'global_accum_on_land_grid = ', global_accum_on_land_grid + write(logunit,*) 'global_accum_on_glc_grid = ', global_accum_on_glc_grid write(logunit,*) 'accum_renorm_factor = ', accum_renorm_factor write(logunit,*) 'ablat_renorm_factor = ', ablat_renorm_factor endif @@ -1424,6 +1605,11 @@ function prep_glc_get_l2gacc_lx_cnt() prep_glc_get_l2gacc_lx_cnt => l2gacc_lx_cnt end function prep_glc_get_l2gacc_lx_cnt + function prep_glc_get_l2gacc_lx_cnt_avg() + integer, pointer :: prep_glc_get_l2gacc_lx_cnt_avg + prep_glc_get_l2gacc_lx_cnt_avg => l2gacc_lx_cnt_avg + end function prep_glc_get_l2gacc_lx_cnt_avg + function prep_glc_get_o2x_gx() type(mct_aVect), pointer :: prep_glc_get_o2x_gx(:) prep_glc_get_o2x_gx => o2x_gx(:) @@ -1449,15 +1635,15 @@ function prep_glc_get_mapper_Fl2g() prep_glc_get_mapper_Fl2g => mapper_Fl2g end function prep_glc_get_mapper_Fl2g - function prep_glc_get_mapper_So2g() - type(seq_map), pointer :: prep_glc_get_mapper_So2g - prep_glc_get_mapper_So2g=> mapper_So2g - end function prep_glc_get_mapper_So2g + function prep_glc_get_mapper_So2g_shelf() + type(seq_map), pointer :: prep_glc_get_mapper_So2g_shelf + prep_glc_get_mapper_So2g_shelf=> mapper_So2g_shelf + end function prep_glc_get_mapper_So2g_shelf - function prep_glc_get_mapper_Fo2g() - type(seq_map), pointer :: prep_glc_get_mapper_Fo2g - prep_glc_get_mapper_Fo2g=> mapper_Fo2g - end function prep_glc_get_mapper_Fo2g + function prep_glc_get_mapper_Fo2g_shelf() + type(seq_map), pointer :: prep_glc_get_mapper_Fo2g_shelf + prep_glc_get_mapper_Fo2g_shelf=> mapper_Fo2g_shelf + end function prep_glc_get_mapper_Fo2g_shelf !*********************************************************************** ! diff --git a/driver-mct/main/seq_diag_mct.F90 b/driver-mct/main/seq_diag_mct.F90 index 8534008f9be7..be9c3a1426ce 100644 --- a/driver-mct/main/seq_diag_mct.F90 +++ b/driver-mct/main/seq_diag_mct.F90 @@ -48,6 +48,9 @@ module seq_diag_mct use shr_reprosum_mod, only : shr_reprosum_calc use seq_diagBGC_mct, only : seq_diagBGC_preprint_mct, seq_diagBGC_print_mct + use prep_glc_mod, only : prep_glc_get_l2gacc_lx_cnt_avg + use glc_elevclass_mod, only: glc_get_num_elevation_classes + implicit none save private @@ -140,42 +143,45 @@ module seq_diag_mct integer(in),parameter :: f_hsen =10 ! heat : sensible integer(in),parameter :: f_hpolar =11 ! heat : AIS imbalance integer(in),parameter :: f_hh2ot =12 ! heat : water temperature - integer(in),parameter :: f_wfrz =13 ! water: freezing - integer(in),parameter :: f_wmelt =14 ! water: melting - integer(in),parameter :: f_wrain =15 ! water: precip, liquid - integer(in),parameter :: f_wsnow =16 ! water: precip, frozen - integer(in),parameter :: f_wpolar =17 ! water: AIS imbalance - integer(in),parameter :: f_wevap =18 ! water: evaporation - integer(in),parameter :: f_wroff =19 ! water: runoff/flood - integer(in),parameter :: f_wioff =20 ! water: frozen runoff - integer(in),parameter :: f_wirrig =21 ! water: irrigation - integer(in),parameter :: f_wfrz_16O =22 ! water: freezing - integer(in),parameter :: f_wmelt_16O =23 ! water: melting - integer(in),parameter :: f_wrain_16O =24 ! water: precip, liquid - integer(in),parameter :: f_wsnow_16O =25 ! water: precip, frozen - integer(in),parameter :: f_wevap_16O =26 ! water: evaporation - integer(in),parameter :: f_wroff_16O =27 ! water: runoff/flood - integer(in),parameter :: f_wioff_16O =28 ! water: frozen runoff - integer(in),parameter :: f_wfrz_18O =29 ! water: freezing - integer(in),parameter :: f_wmelt_18O =30 ! water: melting - integer(in),parameter :: f_wrain_18O =31 ! water: precip, liquid - integer(in),parameter :: f_wsnow_18O =32 ! water: precip, frozen - integer(in),parameter :: f_wevap_18O =33 ! water: evaporation - integer(in),parameter :: f_wroff_18O =34 ! water: runoff/flood - integer(in),parameter :: f_wioff_18O =35 ! water: frozen runoff - integer(in),parameter :: f_wfrz_HDO =36 ! water: freezing - integer(in),parameter :: f_wmelt_HDO =37 ! water: melting - integer(in),parameter :: f_wrain_HDO =38 ! water: precip, liquid - integer(in),parameter :: f_wsnow_HDO =39 ! water: precip, frozen - integer(in),parameter :: f_wevap_HDO =40 ! water: evaporation - integer(in),parameter :: f_wroff_HDO =41 ! water: runoff/flood - integer(in),parameter :: f_wioff_HDO =42 ! water: frozen runoff + integer(in),parameter :: f_hgsmb =13 ! heat : Greenland ice sheet surface mass balance + integer(in),parameter :: f_wfrz =14 ! water: freezing + integer(in),parameter :: f_wmelt =15 ! water: melting + integer(in),parameter :: f_wrain =16 ! water: precip, liquid + integer(in),parameter :: f_wsnow =17 ! water: precip, frozen + integer(in),parameter :: f_wpolar =18 ! water: AIS imbalance + integer(in),parameter :: f_wgsmb =19 ! water: Greenland ice sheet surface mass balance + integer(in),parameter :: f_wevap =20 ! water: evaporation + integer(in),parameter :: f_wroff =21 ! water: runoff/flood + integer(in),parameter :: f_wioff =22 ! water: frozen runoff + integer(in),parameter :: f_wirrig =23 ! water: irrigation + integer(in),parameter :: f_wfrz_16O =24 ! water: freezing + integer(in),parameter :: f_wmelt_16O =25 ! water: melting + integer(in),parameter :: f_wrain_16O =26 ! water: precip, liquid + integer(in),parameter :: f_wsnow_16O =27 ! water: precip, frozen + integer(in),parameter :: f_wevap_16O =28 ! water: evaporation + integer(in),parameter :: f_wroff_16O =29 ! water: runoff/flood + integer(in),parameter :: f_wioff_16O =30 ! water: frozen runoff + integer(in),parameter :: f_wfrz_18O =31 ! water: freezing + integer(in),parameter :: f_wmelt_18O =32 ! water: melting + integer(in),parameter :: f_wrain_18O =33 ! water: precip, liquid + integer(in),parameter :: f_wsnow_18O =34 ! water: precip, frozen + integer(in),parameter :: f_wevap_18O =35 ! water: evaporation + integer(in),parameter :: f_wroff_18O =36 ! water: runoff/flood + integer(in),parameter :: f_wioff_18O =37 ! water: frozen runoff + integer(in),parameter :: f_wfrz_HDO =38 ! water: freezing + integer(in),parameter :: f_wmelt_HDO =39 ! water: melting + integer(in),parameter :: f_wrain_HDO =40 ! water: precip, liquid + integer(in),parameter :: f_wsnow_HDO =41 ! water: precip, frozen + integer(in),parameter :: f_wevap_HDO =42 ! water: evaporation + integer(in),parameter :: f_wroff_HDO =43 ! water: runoff/flood + integer(in),parameter :: f_wioff_HDO =44 ! water: frozen runoff integer(in),parameter :: f_size = f_wioff_HDO ! Total array size of all elements integer(in),parameter :: f_a = f_area ! 1st index for area integer(in),parameter :: f_a_end = f_area ! last index for area integer(in),parameter :: f_h = f_hfrz ! 1st index for heat - integer(in),parameter :: f_h_end = f_hh2ot ! Last index for heat + !integer(in),parameter :: f_h_end = f_hh2ot ! Last index for heat + integer(in),parameter :: f_h_end = f_hgsmb ! Last index for heat integer(in),parameter :: f_w = f_wfrz ! 1st index for water integer(in),parameter :: f_w_end = f_wirrig ! Last index for water integer(in),parameter :: f_16O = f_wfrz_16O ! 1st index for 16O water isotope @@ -189,8 +195,10 @@ module seq_diag_mct (/' area',' hfreeze',' hmelt',' hnetsw',' hlwdn', & ' hlwup',' hlatvap',' hlatfus',' hiroff',' hsen', & - ' hpolar',' hh2otemp',' wfreeze',' wmelt',' wrain', & - ' wsnow',' wpolar',' wevap',' wrunoff',' wfrzrof', & +! ' hpolar',' hh2otemp',' wfreeze',' wmelt',' wrain', & + ' hpolar',' hh2otemp',' hgsmb',' wfreeze',' wmelt',' wrain', & +! ' wsnow',' wpolar',' wevap',' wrunoff',' wfrzrof', & + ' wsnow',' wpolar',' wgsmb',' wevap',' wrunoff',' wfrzrof', & ' wirrig', & ' wfreeze_16O',' wmelt_16O',' wrain_16O',' wsnow_16O', & ' wevap_16O',' wrunoff_16O',' wfrzrof_16O', & @@ -262,6 +270,9 @@ module seq_diag_mct integer :: index_l2x_Flrl_irrig integer :: index_l2x_Flrl_wslake + integer :: index_x2l_Sg_icemask + integer, allocatable :: index_l2x_Flgl_qice(:) + integer, allocatable :: index_x2l_Sg_ice_covered(:) integer :: index_x2l_Faxa_lwdn integer :: index_x2l_Faxa_rainc @@ -338,6 +349,9 @@ module seq_diag_mct integer :: index_g2x_Fogg_rofi integer :: index_g2x_Figg_rofi + integer :: index_x2g_Flgl_qice + integer :: index_g2x_Sg_icemask + integer :: index_x2o_Foxx_rofl_16O integer :: index_x2o_Foxx_rofi_16O integer :: index_x2o_Foxx_rofl_18O @@ -434,6 +448,9 @@ module seq_diag_mct integer :: index_x2i_Faxa_snow_18O integer :: index_x2i_Faxa_snow_HDO + integer :: glc_nec + integer :: l2gacc_lx_cnt_avg + !=============================================================================== contains !=============================================================================== @@ -866,7 +883,7 @@ subroutine seq_diag_lnd_mct( lnd, frac_l, infodata, do_l2x, do_x2l) type(mct_aVect), pointer :: l2x_l ! model to drv bundle type(mct_aVect), pointer :: x2l_l ! drv to model bundle type(mct_ggrid), pointer :: dom_l - integer(in) :: n,ic,nf,ip ! generic index + integer(in) :: n,ic,nf,ip ! generic index integer(in) :: kArea ! index of area field in aVect integer(in) :: kl ! fraction indices integer(in) :: lSize ! size of aVect @@ -874,6 +891,13 @@ subroutine seq_diag_lnd_mct( lnd, frac_l, infodata, do_l2x, do_x2l) logical,save :: first_time = .true. logical,save :: flds_wiso_lnd = .false. + real(r8) :: l2x_Flgl_qice_col_sum ! for summing fluxes over no. of elev. classes + real(r8) :: effective_area + + character(len=64) :: name + character(len= 2) :: cnum + integer(in) :: num + !----- formats ----- character(*),parameter :: subName = '(seq_diag_lnd_mct) ' @@ -894,6 +918,15 @@ subroutine seq_diag_lnd_mct( lnd, frac_l, infodata, do_l2x, do_x2l) kArea = mct_aVect_indexRA(dom_l%data,afldname) kl = mct_aVect_indexRA(frac_l,lfrinname) + ! get number of elevation classes and allocate relevant sets of indices + glc_nec = glc_get_num_elevation_classes() + if (glc_nec.ge.1) then + if (first_time) then + allocate(index_l2x_Flgl_qice(0:glc_nec)) + allocate(index_x2l_Sg_ice_covered(0:glc_nec)) + end if + end if + if (present(do_l2x)) then if (first_time) then index_l2x_Fall_swnet = mct_aVect_indexRA(l2x_l,'Fall_swnet') @@ -909,6 +942,17 @@ subroutine seq_diag_lnd_mct( lnd, frac_l, infodata, do_l2x, do_x2l) index_l2x_Flrl_irrig = mct_aVect_indexRA(l2x_l,'Flrl_irrig', perrWith='quiet') index_l2x_Flrl_wslake = mct_aVect_indexRA(l2x_l,'Flrl_wslake') + if (glc_nec.ge.1) then + index_x2l_Sg_icemask = mct_avect_indexRA(x2l_l,'Sg_icemask') + do num=0,glc_nec + write(cnum,'(i2.2)') num + name = 'Flgl_qice' // cnum + index_l2x_Flgl_qice(num) = mct_avect_indexRA(l2x_l,trim(name)) + name = 'Sg_ice_covered' // cnum + index_x2l_Sg_ice_covered(num) = mct_avect_indexRA(x2l_l,trim(name)) + end do + end if + index_l2x_Fall_evap_16O = mct_aVect_indexRA(l2x_l,'Fall_evap_16O',perrWith='quiet') if ( index_l2x_Fall_evap_16O /= 0 ) flds_wiso_lnd = .true. if ( flds_wiso_lnd )then @@ -942,7 +986,19 @@ subroutine seq_diag_lnd_mct( lnd, frac_l, infodata, do_l2x, do_x2l) if (index_l2x_Flrl_irrig /= 0) then nf = f_wroff ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) - ca_l*l2x_l%rAttr(index_l2x_Flrl_irrig,n) end if - nf = f_wioff ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) - ca_l*l2x_l%rAttr(index_l2x_Flrl_rofi,n) + nf = f_wioff ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) - ca_l*l2x_l%rAttr(index_l2x_Flrl_rofi,n) ! contribution from land ice calving currently zero + + l2x_Flgl_qice_col_sum = 0.0d0 + if (glc_nec.ge.1) then + effective_area = min(frac_l%rAttr(kl,n),x2l_l%rAttr(index_x2l_Sg_icemask,n)) * dom_l%data%rAttr(kArea,n) + do num=0,glc_nec + ! sums the contributions from fluxes in each set of elevation classes + ! RHS product is flux times fraction of area in specific elevation class times land cell area + l2x_Flgl_qice_col_sum = l2x_Flgl_qice_col_sum + l2x_l%rAttr(index_l2x_Flgl_qice(num),n) * & + x2l_l%rAttr(index_x2l_Sg_ice_covered(num),n) * effective_area + end do + end if + nf = f_wgsmb ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) - l2x_Flgl_qice_col_sum if ( flds_wiso_lnd )then nf = f_wevap_16O; @@ -976,7 +1032,14 @@ subroutine seq_diag_lnd_mct( lnd, frac_l, infodata, do_l2x, do_x2l) ca_l*l2x_l%rAttr(index_l2x_Flrl_rofi_HDO,n) end if end do - budg_dataL(f_hioff,ic,ip) = -budg_dataL(f_wioff,ic,ip)*shr_const_latice + + budg_dataL(f_hioff,ic,ip) = -budg_dataL(f_wioff,ic,ip)*shr_const_latice ! contribution from land ice calving currently zero + budg_dataL(f_hgsmb,ic,ip) = budg_dataL(f_wgsmb,ic,ip)*shr_const_latice + + ! Nneeded? Not sure if / when these should be deallocated + !deallocate(index_l2x_Flgl_qice(0:glc_nec)) + !deallocate(index_x2l_Sg_ice_covered(0:glc_nec)) + end if if (present(do_x2l)) then @@ -1252,15 +1315,18 @@ end subroutine seq_diag_rof_mct ! Compute global glc input/output flux diagnostics ! ! !REVISION HISTORY: - ! 2008-jul-10 - T. Craig - update + ! 2008-Jul-10 - T. Craig - update + ! 2024-Dec-06 - S. Price, J. Wolfe - update ! ! !INTERFACE: ------------------------------------------------------------------ - subroutine seq_diag_glc_mct( glc, frac_g, infodata) + subroutine seq_diag_glc_mct( glc, frac_g, infodata, do_x2g, do_g2x ) type(component_type) , intent(in) :: glc ! component type for instance1 - type(mct_aVect) , intent(in) :: frac_g ! frac bundle + type(mct_aVect) , intent(in) :: frac_g ! frac bundle (may not be used / needed here) type(seq_infodata_type) , intent(in) :: infodata + logical , intent(in), optional :: do_x2g + logical , intent(in), optional :: do_g2x !EOP @@ -1269,11 +1335,13 @@ subroutine seq_diag_glc_mct( glc, frac_g, infodata) type(mct_aVect), pointer :: x2g_g type(mct_ggrid), pointer :: dom_g integer(in) :: n,ic,nf,ip ! generic index - integer(in) :: kArea ! index of area field in aVect - integer(in) :: lSize ! size of aVect - real(r8) :: ca_g ! area of a grid cell + integer(in) :: kArea ! index of area field in aVect + integer(in) :: lSize ! size of aVect + real(r8) :: ca_g ! area of a grid cell logical,save :: first_time = .true. + integer,save :: smb_vector_length,calving_vector_length + !----- formats ----- character(*),parameter :: subName = '(seq_diag_glc_mct) ' @@ -1289,23 +1357,61 @@ subroutine seq_diag_glc_mct( glc, frac_g, infodata) g2x_g => component_get_c2x_cx(glc) x2g_g => component_get_x2c_cx(glc) - if (first_time) then - index_g2x_Fogg_rofl = mct_aVect_indexRA(g2x_g,'Fogg_rofl') - index_g2x_Fogg_rofi = mct_aVect_indexRA(g2x_g,'Fogg_rofi') - index_g2x_Figg_rofi = mct_aVect_indexRA(g2x_g,'Figg_rofi') - end if - ip = p_inst - ic = c_glc_gs - kArea = mct_aVect_indexRA(dom_g%data,afldname) - lSize = mct_avect_lSize(g2x_g) - do n=1,lSize - ca_g = dom_g%data%rAttr(kArea,n) - nf = f_wroff; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) - ca_g*g2x_g%rAttr(index_g2x_Fogg_rofl,n) - nf = f_wioff; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) - ca_g*g2x_g%rAttr(index_g2x_Fogg_rofi,n) & - - ca_g*g2x_g%rAttr(index_g2x_Figg_rofi,n) - end do - budg_dataL(f_hioff,ic,ip) = -budg_dataL(f_wioff,ic,ip)*shr_const_latice + + if( present(do_g2x))then ! do fields from glc to coupler (g2x_) + + if (first_time) then + + calving_vector_length = 0 + + index_g2x_Fogg_rofl = mct_aVect_indexRA(g2x_g,'Fogg_rofl') + index_g2x_Fogg_rofi = mct_aVect_indexRA(g2x_g,'Fogg_rofi') + index_g2x_Figg_rofi = mct_aVect_indexRA(g2x_g,'Figg_rofi') + + end if + + ic = c_glc_gr + kArea = mct_aVect_indexRA(dom_g%data,afldname) + lSize = mct_avect_lSize(g2x_g) + + do n=1,lSize + ca_g = dom_g%data%rAttr(kArea,n) + nf = f_wroff; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) - ca_g*g2x_g%rAttr(index_g2x_Fogg_rofl,n) + nf = f_wioff; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) - ca_g*g2x_g%rAttr(index_g2x_Fogg_rofi,n) & + - ca_g*g2x_g%rAttr(index_g2x_Figg_rofi,n) + end do + + budg_dataL(f_hioff,ic,ip) = -budg_dataL(f_wioff,ic,ip)*shr_const_latice + + endif ! end do fields from glc to coupler (g2x_) + + if( present(do_x2g))then ! do fields from coupler to glc (x2g_) + + if (first_time) then + + index_x2g_Flgl_qice = mct_aVect_indexRA(x2g_g,'Flgl_qice') + index_g2x_Sg_icemask = mct_avect_indexRA(g2x_g,'Sg_icemask') + + end if + + l2gacc_lx_cnt_avg = prep_glc_get_l2gacc_lx_cnt_avg() ! counter for how many times SMB flux accumulation has occured + ic = c_glc_gs + kArea = mct_aVect_indexRA(dom_g%data,afldname) + lSize = mct_avect_lSize(x2g_g) + + do n=1,lSize + ca_g = dom_g%data%rAttr(kArea,n)*g2x_g%rAttr(index_g2x_Sg_icemask,n) + nf = f_wgsmb; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + ca_g*x2g_g%rAttr(index_x2g_Flgl_qice,n) + end do + + budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) * l2gacc_lx_cnt_avg + + budg_dataL(f_hgsmb,ic,ip) = budg_dataL(f_wgsmb,ic,ip)*shr_const_latice + + smb_vector_length = smb_vector_length +lSize + + end if ! end do fields from coupler to glc (x2g_) first_time = .false. diff --git a/driver-mct/main/seq_rest_mod.F90 b/driver-mct/main/seq_rest_mod.F90 index 0ad62de966f4..536602783313 100644 --- a/driver-mct/main/seq_rest_mod.F90 +++ b/driver-mct/main/seq_rest_mod.F90 @@ -519,6 +519,8 @@ subroutine seq_rest_write(EClock_d, seq_SyncClock, infodata, & call seq_io_write(rest_file,rvar,'seq_infodata_precip_fact',whead=whead,wdata=wdata) call seq_infodata_GetData(infodata,case_name=cvar) call seq_io_write(rest_file,trim(cvar),'seq_infodata_case_name',whead=whead,wdata=wdata) + call seq_infodata_GetData(infodata,rmean_rmv_ice_runoff=rvar) + call seq_io_write(rest_file,rvar,'seq_infodata_rmean_rmv_ice_runoff',whead=whead,wdata=wdata) call seq_timemgr_EClockGetData( EClock_d, start_ymd=ivar) call seq_io_write(rest_file,ivar,'seq_timemgr_start_ymd',whead=whead,wdata=wdata) diff --git a/driver-mct/shr/seq_flds_mod.F90 b/driver-mct/shr/seq_flds_mod.F90 index dbfba0889d0c..d0f46ed36e2a 100644 --- a/driver-mct/shr/seq_flds_mod.F90 +++ b/driver-mct/shr/seq_flds_mod.F90 @@ -212,7 +212,8 @@ module seq_flds_mod character(CXX) :: seq_flds_g2o_ice_fluxes character(CXX) :: seq_flds_x2g_states character(CXX) :: seq_flds_x2g_states_from_lnd - character(CXX) :: seq_flds_x2g_states_from_ocn + character(CXX) :: seq_flds_x2g_shelf_states_from_ocn + character(CXX) :: seq_flds_x2g_tf_states_from_ocn character(CXX) :: seq_flds_x2g_fluxes character(CXX) :: seq_flds_x2g_fluxes_from_lnd @@ -347,7 +348,8 @@ subroutine seq_flds_set(nmlfile, ID, infodata) character(CXX) :: g2o_ice_fluxes = '' character(CXX) :: x2g_states = '' character(CXX) :: x2g_states_from_lnd = '' - character(CXX) :: x2g_states_from_ocn = '' + character(CXX) :: x2g_shelf_states_from_ocn = '' + character(CXX) :: x2g_tf_states_from_ocn = '' character(CXX) :: x2g_fluxes = '' character(CXX) :: x2g_fluxes_from_lnd = '' character(CXX) :: xao_albedo = '' @@ -380,11 +382,12 @@ subroutine seq_flds_set(nmlfile, ID, infodata) logical :: flds_bgc_oi logical :: flds_wiso logical :: flds_polar + logical :: flds_tf integer :: glc_nec namelist /seq_cplflds_inparm/ & - flds_co2a, flds_co2b, flds_co2c, flds_co2_dmsa, flds_wiso, flds_polar, glc_nec, & - ice_ncat, seq_flds_i2o_per_cat, flds_bgc_oi, & + flds_co2a, flds_co2b, flds_co2c, flds_co2_dmsa, flds_wiso, flds_polar, flds_tf, & + glc_nec, ice_ncat, seq_flds_i2o_per_cat, flds_bgc_oi, & nan_check_component_fields, rof_heat, atm_flux_method, atm_gustiness, & rof2ocn_nutrients, lnd_rof_two_way, ocn_rof_two_way, rof_sed @@ -418,6 +421,7 @@ subroutine seq_flds_set(nmlfile, ID, infodata) flds_bgc_oi = .false. flds_wiso = .false. flds_polar = .false. + flds_tf = .false. glc_nec = 0 ice_ncat = 1 seq_flds_i2o_per_cat = .false. @@ -452,6 +456,7 @@ subroutine seq_flds_set(nmlfile, ID, infodata) call shr_mpi_bcast(flds_bgc_oi , mpicom) call shr_mpi_bcast(flds_wiso , mpicom) call shr_mpi_bcast(flds_polar , mpicom) + call shr_mpi_bcast(flds_tf , mpicom) call shr_mpi_bcast(glc_nec , mpicom) call shr_mpi_bcast(ice_ncat , mpicom) call shr_mpi_bcast(seq_flds_i2o_per_cat, mpicom) @@ -2938,7 +2943,7 @@ subroutine seq_flds_set(nmlfile, ID, infodata) name = 'So_blt' call seq_flds_add(o2x_states,trim(name)) call seq_flds_add(x2g_states,trim(name)) - call seq_flds_add(x2g_states_from_ocn,trim(name)) + call seq_flds_add(x2g_shelf_states_from_ocn,trim(name)) longname = 'Ice shelf boundary layer ocean temperature' stdname = 'Ice_shelf_boundary_layer_ocean_temperature' units = 'C' @@ -2948,7 +2953,7 @@ subroutine seq_flds_set(nmlfile, ID, infodata) name = 'So_bls' call seq_flds_add(o2x_states,trim(name)) call seq_flds_add(x2g_states,trim(name)) - call seq_flds_add(x2g_states_from_ocn,trim(name)) + call seq_flds_add(x2g_shelf_states_from_ocn,trim(name)) longname = 'Ice shelf boundary layer ocean salinity' stdname = 'Ice_shelf_boundary_layer_ocean_salinity' units = 'psu' @@ -2958,7 +2963,7 @@ subroutine seq_flds_set(nmlfile, ID, infodata) name = 'So_htv' call seq_flds_add(o2x_states,trim(name)) call seq_flds_add(x2g_states,trim(name)) - call seq_flds_add(x2g_states_from_ocn,trim(name)) + call seq_flds_add(x2g_shelf_states_from_ocn,trim(name)) longname = 'Ice shelf ocean heat transfer velocity' stdname = 'Ice_shelf_ocean_heat_transfer_velocity' units = 'm/s' @@ -2968,7 +2973,7 @@ subroutine seq_flds_set(nmlfile, ID, infodata) name = 'So_stv' call seq_flds_add(o2x_states,trim(name)) call seq_flds_add(x2g_states,trim(name)) - call seq_flds_add(x2g_states_from_ocn,trim(name)) + call seq_flds_add(x2g_shelf_states_from_ocn,trim(name)) longname = 'Ice shelf ocean salinity transfer velocity' stdname = 'Ice_shelf_ocean_salinity_transfer_velocity' units = 'm/s' @@ -2978,13 +2983,27 @@ subroutine seq_flds_set(nmlfile, ID, infodata) name = 'So_rhoeff' call seq_flds_add(o2x_states,trim(name)) call seq_flds_add(x2g_states,trim(name)) - call seq_flds_add(x2g_states_from_ocn,trim(name)) + call seq_flds_add(x2g_shelf_states_from_ocn,trim(name)) longname = 'Ocean effective pressure' stdname = 'Ocean_effective_pressure' units = 'Pa' attname = 'So_rhoeff' call metadata_set(attname, longname, stdname, units) + if (flds_tf) then + + name = 'So_tf2d' + call seq_flds_add(o2x_states,trim(name)) + call seq_flds_add(x2g_states,trim(name)) + call seq_flds_add(x2g_tf_states_from_ocn,trim(name)) + longname = 'ocean thermal forcing at predefined critical depth' + stdname = 'ocean_thermal_forcing_at_critical_depth' + units = 'C' + attname = name + call metadata_set(attname, longname, stdname, units) + + end if + name = 'Fogx_qicelo' call seq_flds_add(g2x_fluxes,trim(name)) call seq_flds_add(x2o_fluxes,trim(name)) @@ -3936,7 +3955,8 @@ subroutine seq_flds_set(nmlfile, ID, infodata) seq_flds_g2x_states_to_lnd = trim(g2x_states_to_lnd) seq_flds_x2g_states = trim(x2g_states) seq_flds_x2g_states_from_lnd = trim(x2g_states_from_lnd) - seq_flds_x2g_states_from_ocn = trim(x2g_states_from_ocn) + seq_flds_x2g_shelf_states_from_ocn = trim(x2g_shelf_states_from_ocn) + seq_flds_x2g_tf_states_from_ocn = trim(x2g_tf_states_from_ocn) seq_flds_xao_states = trim(xao_states) seq_flds_xao_albedo = trim(xao_albedo) seq_flds_xao_diurnl = trim(xao_diurnl) @@ -4003,7 +4023,8 @@ subroutine seq_flds_set(nmlfile, ID, infodata) write(logunit,*) subname//': seq_flds_x2g_states= ',trim(seq_flds_x2g_states) write(logunit,*) subname//': seq_flds_x2g_states_from_lnd= ',trim(seq_flds_x2g_states_from_lnd) write(logunit,*) subname//': seq_flds_l2x_states_to_glc= ',trim(seq_flds_l2x_states_to_glc) - write(logunit,*) subname//': seq_flds_x2g_states_from_ocn= ',trim(seq_flds_x2g_states_from_ocn) + write(logunit,*) subname//': seq_flds_x2g_shelf_states_from_ocn= ',trim(seq_flds_x2g_shelf_states_from_ocn) + write(logunit,*) subname//': seq_flds_x2g_tf_states_from_ocn= ',trim(seq_flds_x2g_tf_states_from_ocn) write(logunit,*) subname//': seq_flds_x2g_fluxes= ',trim(seq_flds_x2g_fluxes) write(logunit,*) subname//': seq_flds_x2g_fluxes_from_lnd= ',trim(seq_flds_x2g_fluxes_from_lnd) write(logunit,*) subname//': seq_flds_l2x_fluxes_to_glc= ',trim(seq_flds_l2x_fluxes_to_glc) diff --git a/driver-mct/shr/seq_infodata_mod.F90 b/driver-mct/shr/seq_infodata_mod.F90 index fcc6a21eef1f..be6810b613e9 100644 --- a/driver-mct/shr/seq_infodata_mod.F90 +++ b/driver-mct/shr/seq_infodata_mod.F90 @@ -203,6 +203,7 @@ MODULE seq_infodata_mod logical :: ocn_prognostic ! does component model need input data from driver logical :: ocnrof_prognostic ! does component need rof data logical :: ocn_c2_glcshelf ! will ocn component send data for ice shelf fluxes in driver + logical :: ocn_c2_glctf ! will ocn component send data for thermal forcing in driver logical :: ice_present ! does component model exist logical :: ice_prognostic ! does component model need input data from driver logical :: iceberg_prognostic ! does the ice model support icebergs @@ -250,7 +251,8 @@ MODULE seq_infodata_mod integer(SHR_KIND_IN) :: iac_phase ! iac phase logical :: atm_aero ! atmosphere aerosols logical :: glc_g2lupdate ! update glc2lnd fields in lnd model - real(shr_kind_r8) :: max_cplstep_time ! abort if cplstep time exceeds this value + real(SHR_KIND_R8) :: max_cplstep_time ! abort if cplstep time exceeds this value + real(SHR_KIND_R8) :: rmean_rmv_ice_runoff ! running mean of removed Antarctic ice runoff !--- set from restart file --- character(SHR_KIND_CL) :: rest_case_name ! Short case identification !--- set by driver and may be time varying @@ -761,10 +763,11 @@ SUBROUTINE seq_infodata_Init( infodata, nmlfile, ID, pioid, cpl_tag) infodata%atm_prognostic = .false. infodata%lnd_prognostic = .false. infodata%rof_prognostic = .false. - infodata%rofocn_prognostic = .false. + infodata%rofocn_prognostic = .false. infodata%ocn_prognostic = .false. infodata%ocnrof_prognostic = .false. infodata%ocn_c2_glcshelf = .false. + infodata%ocn_c2_glctf = .false. infodata%ice_prognostic = .false. infodata%glc_prognostic = .false. ! It's safest to assume glc_coupled_fluxes = .true. if it's not set elsewhere, @@ -808,6 +811,7 @@ SUBROUTINE seq_infodata_Init( infodata, nmlfile, ID, pioid, cpl_tag) infodata%atm_aero = .false. infodata%glc_g2lupdate = .false. infodata%glc_valid_input = .true. + infodata%rmean_rmv_ice_runoff = -1.0_SHR_KIND_R8 infodata%max_cplstep_time = max_cplstep_time infodata%model_doi_url = model_doi_url @@ -907,11 +911,13 @@ SUBROUTINE seq_infodata_Init( infodata, nmlfile, ID, pioid, cpl_tag) call seq_io_read(infodata%restart_file,pioid,infodata%nextsw_cday ,'seq_infodata_nextsw_cday') call seq_io_read(infodata%restart_file,pioid,infodata%precip_fact ,'seq_infodata_precip_fact') call seq_io_read(infodata%restart_file,pioid,infodata%rest_case_name,'seq_infodata_case_name') + call seq_io_read(infodata%restart_file,pioid,infodata%rmean_rmv_ice_runoff,'seq_infodata_rmean_rmv_ice_runoff') endif !--- Send from CPLID ROOT to GLOBALID ROOT, use bcast as surrogate call shr_mpi_bcast(infodata%nextsw_cday,mpicom,pebcast=seq_comm_gloroot(CPLID)) call shr_mpi_bcast(infodata%precip_fact,mpicom,pebcast=seq_comm_gloroot(CPLID)) call shr_mpi_bcast(infodata%rest_case_name,mpicom,pebcast=seq_comm_gloroot(CPLID)) + call shr_mpi_bcast(infodata%rmean_rmv_ice_runoff,mpicom,pebcast=seq_comm_gloroot(CPLID)) endif if (seq_comm_iamroot(ID)) then @@ -1004,7 +1010,8 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ atm_present, atm_prognostic, & lnd_present, lnd_prognostic, & rof_present, rof_prognostic, rofocn_prognostic, & - ocn_present, ocn_prognostic, ocnrof_prognostic, ocn_c2_glcshelf, & + ocn_present, ocn_prognostic, ocnrof_prognostic, & + ocn_c2_glcshelf, ocn_c2_glctf, & ice_present, ice_prognostic, & glc_present, glc_prognostic, & iac_present, iac_prognostic, & @@ -1041,7 +1048,8 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ reprosum_use_ddpdd, reprosum_allow_infnan, & reprosum_diffmax, reprosum_recompute, & mct_usealltoall, mct_usevector, max_cplstep_time, model_doi_url, & - glc_valid_input, nlmaps_verbosity, nlmaps_exclude_fields) + glc_valid_input, nlmaps_verbosity, nlmaps_exclude_fields, & + rmean_rmv_ice_runoff) implicit none @@ -1179,6 +1187,7 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ logical, optional, intent(OUT) :: ocn_prognostic logical, optional, intent(OUT) :: ocnrof_prognostic logical, optional, intent(OUT) :: ocn_c2_glcshelf + logical, optional, intent(OUT) :: ocn_c2_glctf logical, optional, intent(OUT) :: ice_present logical, optional, intent(OUT) :: ice_prognostic logical, optional, intent(OUT) :: iceberg_prognostic @@ -1228,6 +1237,7 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ real(shr_kind_r8), optional, intent(out) :: max_cplstep_time character(SHR_KIND_CL), optional, intent(OUT) :: model_doi_url logical, optional, intent(OUT) :: glc_valid_input + real(SHR_KIND_R8), optional, intent(out) :: rmean_rmv_ice_runoff !----- local ----- character(len=*), parameter :: subname = '(seq_infodata_GetData_explicit) ' @@ -1365,6 +1375,7 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ if ( present(ocn_prognostic) ) ocn_prognostic = infodata%ocn_prognostic if ( present(ocnrof_prognostic) ) ocnrof_prognostic = infodata%ocnrof_prognostic if ( present(ocn_c2_glcshelf) ) ocn_c2_glcshelf = infodata%ocn_c2_glcshelf + if ( present(ocn_c2_glctf) ) ocn_c2_glctf = infodata%ocn_c2_glctf if ( present(ice_present) ) ice_present = infodata%ice_present if ( present(ice_prognostic) ) ice_prognostic = infodata%ice_prognostic if ( present(iceberg_prognostic)) iceberg_prognostic = infodata%iceberg_prognostic @@ -1427,6 +1438,7 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ if ( present(model_doi_url) ) model_doi_url = infodata%model_doi_url if ( present(glc_valid_input)) glc_valid_input = infodata%glc_valid_input + if ( present(rmean_rmv_ice_runoff) ) rmean_rmv_ice_runoff = infodata%rmean_rmv_ice_runoff END SUBROUTINE seq_infodata_GetData_explicit @@ -1557,7 +1569,8 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ atm_present, atm_prognostic, & lnd_present, lnd_prognostic, & rof_present, rof_prognostic, rofocn_prognostic, & - ocn_present, ocn_prognostic, ocnrof_prognostic, ocn_c2_glcshelf, & + ocn_present, ocn_prognostic, ocnrof_prognostic, & + ocn_c2_glcshelf, ocn_c2_glctf, & ice_present, ice_prognostic, & glc_present, glc_prognostic, & glc_coupled_fluxes, & @@ -1595,7 +1608,7 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ reprosum_use_ddpdd, reprosum_allow_infnan, & reprosum_diffmax, reprosum_recompute, & mct_usealltoall, mct_usevector, glc_valid_input, & - nlmaps_verbosity, nlmaps_exclude_fields) + nlmaps_verbosity, nlmaps_exclude_fields, rmean_rmv_ice_runoff) implicit none @@ -1732,6 +1745,7 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ logical, optional, intent(IN) :: ocn_prognostic logical, optional, intent(IN) :: ocnrof_prognostic logical, optional, intent(IN) :: ocn_c2_glcshelf + logical, optional, intent(IN) :: ocn_c2_glctf logical, optional, intent(IN) :: ice_present logical, optional, intent(IN) :: ice_prognostic logical, optional, intent(IN) :: iceberg_prognostic @@ -1778,6 +1792,7 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ logical, optional, intent(IN) :: atm_aero ! atm aerosols logical, optional, intent(IN) :: glc_g2lupdate ! update glc2lnd fields in lnd model logical, optional, intent(IN) :: glc_valid_input + real(SHR_KIND_R8), optional, intent(IN) :: rmean_rmv_ice_runoff ! running mean of removed Antarctic ice runoff !EOP @@ -1917,6 +1932,7 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ if ( present(ocn_prognostic) ) infodata%ocn_prognostic = ocn_prognostic if ( present(ocnrof_prognostic)) infodata%ocnrof_prognostic = ocnrof_prognostic if ( present(ocn_c2_glcshelf)) infodata%ocn_c2_glcshelf = ocn_c2_glcshelf + if ( present(ocn_c2_glctf)) infodata%ocn_c2_glctf = ocn_c2_glctf if ( present(ice_present) ) infodata%ice_present = ice_present if ( present(ice_prognostic) ) infodata%ice_prognostic = ice_prognostic if ( present(iceberg_prognostic)) infodata%iceberg_prognostic = iceberg_prognostic @@ -1963,6 +1979,7 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ if ( present(atm_aero) ) infodata%atm_aero = atm_aero if ( present(glc_g2lupdate) ) infodata%glc_g2lupdate = glc_g2lupdate if ( present(glc_valid_input) ) infodata%glc_valid_input = glc_valid_input + if ( present(rmean_rmv_ice_runoff) ) infodata%rmean_rmv_ice_runoff = rmean_rmv_ice_runoff END SUBROUTINE seq_infodata_PutData_explicit @@ -2229,6 +2246,7 @@ subroutine seq_infodata_bcast(infodata,mpicom) call shr_mpi_bcast(infodata%ocn_prognostic, mpicom) call shr_mpi_bcast(infodata%ocnrof_prognostic, mpicom) call shr_mpi_bcast(infodata%ocn_c2_glcshelf, mpicom) + call shr_mpi_bcast(infodata%ocn_c2_glctf, mpicom) call shr_mpi_bcast(infodata%ice_present, mpicom) call shr_mpi_bcast(infodata%ice_prognostic, mpicom) call shr_mpi_bcast(infodata%iceberg_prognostic, mpicom) @@ -2277,6 +2295,7 @@ subroutine seq_infodata_bcast(infodata,mpicom) call shr_mpi_bcast(infodata%glc_valid_input, mpicom) call shr_mpi_bcast(infodata%model_doi_url, mpicom) call shr_mpi_bcast(infodata%constant_zenith_deg, mpicom) + call shr_mpi_bcast(infodata%rmean_rmv_ice_runoff, mpicom) end subroutine seq_infodata_bcast @@ -2515,6 +2534,7 @@ subroutine seq_infodata_Exchange(infodata,ID,type) call shr_mpi_bcast(infodata%ocn_prognostic, mpicom, pebcast=cmppe) call shr_mpi_bcast(infodata%ocnrof_prognostic, mpicom, pebcast=cmppe) call shr_mpi_bcast(infodata%ocn_c2_glcshelf, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%ocn_c2_glctf, mpicom, pebcast=cmppe) call shr_mpi_bcast(infodata%ocn_nx, mpicom, pebcast=cmppe) call shr_mpi_bcast(infodata%ocn_ny, mpicom, pebcast=cmppe) ! dead_comps is true if it's ever set to true @@ -2591,6 +2611,7 @@ subroutine seq_infodata_Exchange(infodata,ID,type) call shr_mpi_bcast(infodata%ocn_prognostic, mpicom, pebcast=cplpe) call shr_mpi_bcast(infodata%ocnrof_prognostic, mpicom, pebcast=cplpe) call shr_mpi_bcast(infodata%ocn_c2_glcshelf, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%ocn_c2_glctf, mpicom, pebcast=cplpe) call shr_mpi_bcast(infodata%ice_present, mpicom, pebcast=cplpe) call shr_mpi_bcast(infodata%ice_prognostic, mpicom, pebcast=cplpe) call shr_mpi_bcast(infodata%iceberg_prognostic, mpicom, pebcast=cplpe) @@ -2617,6 +2638,7 @@ subroutine seq_infodata_Exchange(infodata,ID,type) if (ocn2cplr) then call shr_mpi_bcast(infodata%precip_fact, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%rmean_rmv_ice_runoff, mpicom, pebcast=cmppe) endif if (cpl2r) then @@ -2624,6 +2646,7 @@ subroutine seq_infodata_Exchange(infodata,ID,type) call shr_mpi_bcast(infodata%precip_fact, mpicom, pebcast=cplpe) call shr_mpi_bcast(infodata%glc_g2lupdate, mpicom, pebcast=cplpe) call shr_mpi_bcast(infodata%glc_valid_input, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%rmean_rmv_ice_runoff, mpicom, pebcast=cplpe) endif end subroutine seq_infodata_Exchange @@ -2948,6 +2971,7 @@ SUBROUTINE seq_infodata_print( infodata ) write(logunit,F0L) subname,'ocn_prognostic = ', infodata%ocn_prognostic write(logunit,F0L) subname,'ocnrof_prognostic = ', infodata%ocnrof_prognostic write(logunit,F0L) subname,'ocn_c2_glcshelf = ', infodata%ocn_c2_glcshelf + write(logunit,F0L) subname,'ocn_c2_glctf = ', infodata%ocn_c2_glctf write(logunit,F0L) subname,'ice_present = ', infodata%ice_present write(logunit,F0L) subname,'ice_prognostic = ', infodata%ice_prognostic write(logunit,F0L) subname,'iceberg_prognostic = ', infodata%iceberg_prognostic @@ -2995,6 +3019,7 @@ SUBROUTINE seq_infodata_print( infodata ) write(logunit,F0S) subname,'iac_phase = ', infodata%iac_phase write(logunit,F0L) subname,'glc_g2lupdate = ', infodata%glc_g2lupdate + write(logunit,F0R) subname,'rmean_rmv_ice_runoff = ', infodata%rmean_rmv_ice_runoff ! endif END SUBROUTINE seq_infodata_print diff --git a/driver-moab/cime_config/config_component.xml b/driver-moab/cime_config/config_component.xml index d3685126c61e..a32fb9bd529e 100644 --- a/driver-moab/cime_config/config_component.xml +++ b/driver-moab/cime_config/config_component.xml @@ -1375,6 +1375,14 @@ atm2ocn flux mapping file + + char + idmap_ignore + run_domain + env_run.xml + atm2ice flux mapping file + + char idmap @@ -2620,7 +2628,7 @@ char - netcdf,pnetcdf,netcdf4p,netcdf4c,adios,hdf5,default + netcdf,pnetcdf,netcdf4p,netcdf4c,adios,adiosc,hdf5,default run_pio env_run.xml pio io type diff --git a/driver-moab/cime_config/config_component_e3sm.xml b/driver-moab/cime_config/config_component_e3sm.xml index 044cc1e9f9d6..c16a493f0453 100644 --- a/driver-moab/cime_config/config_component_e3sm.xml +++ b/driver-moab/cime_config/config_component_e3sm.xml @@ -185,6 +185,18 @@ Turn on the passing of polar fields through the coupler + + logical + TRUE,FALSE + FALSE + + TRUE + + run_flags + env_run.xml + Turn on the passing of ocean thermal forcing fields through the coupler + + char minus1p8,linear_salt,mushy @@ -245,6 +257,7 @@ CESM1_MOD CESM1_MOD RASM_OPTION1 + RASM_OPTION2 run_coupling env_run.xml @@ -275,7 +288,7 @@ none CO2C CO2A - CO2A + CO2A CO2A CO2A CO2A @@ -383,6 +396,10 @@ 24 48 48 + 180 + 360 + 720 + 1440 48 48 96 @@ -407,6 +424,7 @@ 144 432 864 + 864 144 96 48 @@ -446,6 +464,7 @@ 1 $ATM_NCPL 48 + $ATM_NCPL $ATM_NCPL 12 96 @@ -747,8 +766,13 @@ 312.821 388.717 388.717 - 0.000001 - 0.000001 + 284.317 + 284.317 + 284.317 + 284.317 + 284.317 + 284.317 + 0.000001 284.317 284.317 284.317 @@ -851,6 +875,7 @@ 1 2 + 2 shr_dust_nl env_run.xml diff --git a/driver-moab/main/cime_comp_mod.F90 b/driver-moab/main/cime_comp_mod.F90 index 265f8e11338e..59fa2daf4080 100644 --- a/driver-moab/main/cime_comp_mod.F90 +++ b/driver-moab/main/cime_comp_mod.F90 @@ -2896,10 +2896,12 @@ subroutine cime_run() num_moab_exports = num_moab_exports + 1! this is moab clock used for debugging call seq_timemgr_clockAdvance( seq_SyncClock, force_stop, force_stop_ymd, force_stop_tod) call seq_timemgr_EClockGetData(EClock_d, stepno=cur_step_no) +#ifdef MOABDEBUG if (iamroot_CPLID) then write(logunit,*) ' num_moab_exports , cur_step_no ',num_moab_exports, cur_step_no call shr_sys_flush(logunit) endif +#endif call seq_timemgr_EClockGetData( EClock_d, curr_ymd=ymd, curr_tod=tod) call shr_cal_date2ymd(ymd,year,month,day) stop_alarm = seq_timemgr_alarmIsOn(EClock_d,seq_timemgr_alarm_stop) diff --git a/driver-moab/main/component_mod.F90 b/driver-moab/main/component_mod.F90 index 88d152a95685..c7a8f51b99ed 100644 --- a/driver-moab/main/component_mod.F90 +++ b/driver-moab/main/component_mod.F90 @@ -471,6 +471,8 @@ subroutine component_init_aream(infodata, rof_c2_ocn, samegrid_ao, samegrid_al, ! character(1024) :: domain_file ! file containing domain info (set my input) use seq_comm_mct, only: mboxid ! iMOAB id for MPAS ocean migrated mesh to coupler pes use seq_comm_mct, only: mbaxid ! iMOAB id for atm migrated mesh to coupler pes + use seq_comm_mct, only: mbrxid ! iMOAB id for rof migrated mesh to coupler pes + use seq_comm_mct, only: mb_rof_aream_computed #endif ! ! Arguments @@ -527,6 +529,9 @@ subroutine component_init_aream(infodata, rof_c2_ocn, samegrid_ao, samegrid_al, dom_s%data%rAttr(km,:) = dom_s%data%rAttr(ka,:) #ifdef HAVE_MOAB + ! TODO should actually compute aream from mesh model + ! we do a lot of unnecessary gymnastics, and very inefficient, because we have a + ! different distribution compared to mct source grid atm tagtype = 1 ! dense, double tagname='aream'//C_NULL_CHAR nloc = mct_avect_lsize(dom_s%data) @@ -542,6 +547,8 @@ subroutine component_init_aream(infodata, rof_c2_ocn, samegrid_ao, samegrid_al, write(logunit,*) subname,' error in setting the aream tag on atm ' call shr_sys_abort(subname//' ERROR in setting aream tag on atm ') endif + deallocate(gids) + deallocate(data1) ! project now aream on ocean (from atm) #endif call seq_map_map(mapper_Fa2o, av_s=dom_s%data, av_d=dom_d%data, fldlist='aream') @@ -597,7 +604,40 @@ subroutine component_init_aream(infodata, rof_c2_ocn, samegrid_ao, samegrid_al, gsmap_s=gsmap_s, av_s=dom_s%data, avfld_s='aream', filefld_s='area_a', & string='rof2ocn ice aream initialization') call t_stopf('CPL:seq_map_readdata-rof2ocn_ice') - + ! this should be more efficient if we just compute aream on coupler side, from actual mesh that we have + ! we need to expose that method in iMOAB, which is local + ! what we do here, we get aream from the domain dom_rx, we just filled it above, with readdata + if(.not. mb_rof_aream_computed) then + + ! we do a lot of unnecessary gymnastics, and very inefficient, because we have a + ! different distribution compared to mct source grid atm + tagtype = 1 ! dense, double + tagname='aream'//C_NULL_CHAR + nloc = mct_avect_lsize(dom_s%data) + allocate(data1(nloc)) + km = mct_aVect_indexRa(dom_s%data, "aream" ) + data1 = dom_s%data%rAttr(km,:) + ent_type = 1 ! element dense double tags + allocate(gids(nloc)) + gids = dom_s%data%iAttr(mct_aVect_indexIA(dom_s%data,"GlobGridNum"),:) + ! ! now set data on the coupler side too + ierr = iMOAB_SetDoubleTagStorageWithGid ( mbrxid, tagname, nloc, ent_type, & + data1, gids) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in setting the aream tag on rof ' + call shr_sys_abort(subname//' ERROR in setting aream tag on rof ') + endif + deallocate(gids) + deallocate(data1) +#ifdef MOABDEBUG + ierr = iMOAB_WriteMesh(mbrxid, trim('recRofWithAream.h5m'//C_NULL_CHAR), & + trim(';PARALLEL=WRITE_PART'//C_NULL_CHAR)) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in writing rof mesh coupler ' + call shr_sys_abort(subname//' ERROR in writing rof mesh coupler ') + endif +#endif + endif endif end if @@ -748,6 +788,7 @@ subroutine component_init_areacor_moab (comp, mbccid, mbcxid, seq_flds_c2x_fluxe lsize = comp(1)%mblsize allocate(areas (lsize, 3)) ! lsize is along grid; read mask too allocate(factors (lsize, 2)) + factors = 1.0 ! initialize with 1.0 all factors; then maybe correct them ! get areas tagname='area:aream:mask'//C_NULL_CHAR arrsize = 3 * lsize diff --git a/driver-moab/main/cplcomp_exchange_mod.F90 b/driver-moab/main/cplcomp_exchange_mod.F90 index 76737ad31419..259f6953cff9 100644 --- a/driver-moab/main/cplcomp_exchange_mod.F90 +++ b/driver-moab/main/cplcomp_exchange_mod.F90 @@ -24,7 +24,6 @@ module cplcomp_exchange_mod use seq_comm_mct, only : mhpgid ! iMOAB app id for atm pgx grid, on atm pes use seq_comm_mct, only : atm_pg_active ! flag if PG mesh instanced use seq_comm_mct, only : mlnid , mblxid ! iMOAB app id for land , on land pes and coupler pes - use seq_comm_mct, only : mb_land_mesh ! if true mesh for land use seq_comm_mct, only : mphaid ! iMOAB app id for phys atm; comp atm is 5, phys 5+200 use seq_comm_mct, only : MPSIID, mbixid ! sea-ice on comp pes and on coupler pes use seq_comm_mct, only : mrofid, mbrxid ! iMOAB id of moab rof app on comp pes and on coupler too @@ -1520,16 +1519,12 @@ subroutine cplcomp_moab_Init(infodata,comp) ! we are now on joint pes, compute comm graph between lnd and coupler model typeA = 2 ! point cloud on component PEs, land - if (mb_land_mesh) then - typeA = 3 - endif typeB = 3 ! full mesh on coupler pes, we just read it if (mlnid >= 0) then ierr = iMOAB_GetMeshInfo ( mlnid, nvert, nvise, nbl, nsurf, nvisBC ) comp%mbApCCid = mlnid ! phys atm comp%mbGridType = typeA - 2 ! 0 or 1, pc or cells comp%mblsize = nvert(1) ! vertices - if (mb_land_mesh) comp%mblsize = nvise(1) ! cells endif ierr = iMOAB_ComputeCommGraph( mlnid, mblxid, mpicom_join, mpigrp_old, mpigrp_cplid, & typeA, typeB, id_old, id_join) diff --git a/driver-moab/main/prep_lnd_mod.F90 b/driver-moab/main/prep_lnd_mod.F90 index 44ae5490e2c6..097782f330d6 100644 --- a/driver-moab/main/prep_lnd_mod.F90 +++ b/driver-moab/main/prep_lnd_mod.F90 @@ -1,6 +1,6 @@ module prep_lnd_mod - use shr_kind_mod , only: r8 => SHR_KIND_R8 + use shr_kind_mod , only: R8 => SHR_KIND_R8 use shr_kind_mod , only: cs => SHR_KIND_CS use shr_kind_mod , only: cl => SHR_KIND_CL use shr_kind_mod , only: cxx => SHR_KIND_CXX @@ -18,6 +18,7 @@ module prep_lnd_mod use seq_comm_mct, only: mbrxid ! iMOAB id of moab rof on coupler pes (FV now) use seq_comm_mct, only: mbintxal ! iMOAB id for intx mesh between atm and lnd use seq_comm_mct, only: mbintxrl ! iMOAB id for intx mesh between river and land + use seq_comm_mct, only: mb_rof_aream_computed ! signal use seq_comm_mct, only: mbaxid ! iMOAB id for atm migrated mesh to coupler pes use seq_comm_mct, only: atm_pg_active ! whether the atm uses FV mesh or not ; made true if fv_nphys > 0 @@ -162,7 +163,7 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln integer nrflds ! number of rof fields projected on land integer arrsize ! for setting the r2x fields on land to 0 integer ent_type ! for setting tags - real (kind=r8) , allocatable :: tmparray (:) ! used to set the r2x fields to 0 + real (kind=R8) , allocatable :: tmparray (:) ! used to set the r2x fields to 0 #endif character(*), parameter :: subname = '(prep_lnd_init)' @@ -222,7 +223,6 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln call seq_map_init_rcfile(mapper_Fr2l, rof(1), lnd(1), & 'seq_maps.rc','rof2lnd_fmapname:','rof2lnd_fmaptype:',samegrid_lr, & string='mapper_Fr2l initialization',esmf_map=esmf_map_flag) - end if ! symmetric of l2r, from prep_rof #ifdef HAVE_MOAB ! Call moab intx only if land and river are init in moab @@ -328,6 +328,9 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln fNoBubble, monotonicity, volumetric, fInverseDistanceMap, & noConserve, validate, & trim(dofnameS), trim(dofnameT) ) + + ! signal that the aream for rof has been computed + mb_rof_aream_computed = .true. if (ierr .ne. 0) then write(logunit,*) subname,' error in computing rl weights ' call shr_sys_abort(subname//' ERROR in computing rl weights ') @@ -370,7 +373,7 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln arrsize = nrflds*mlsize allocate (tmparray(arrsize)) ! mlsize is the size of local land ! do we need to zero out others or just river ? - tmparray = 0._r8 + tmparray = 0._R8 ierr = iMOAB_SetDoubleTagStorage(mblxid, tagname, arrsize , ent_type, tmparray) if (ierr .ne. 0) then write(logunit,*) subname,' cant zero out r2x tags on land' @@ -381,6 +384,7 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln end if ! if ((mbrxid .ge. 0) .and. (mblxid .ge. 0)) ! endif HAVE_MOAB #endif + end if call shr_sys_flush(logunit) if (atm_c2_lnd) then @@ -693,7 +697,7 @@ subroutine prep_lnd_mrg_moab (infodata) #endif #ifdef MOABCOMP character(CXX) :: tagname, mct_field - real(r8) :: difference + real(R8) :: difference type(mct_list) :: temp_list integer :: size_list, index_list, ent_type type(mct_string) :: mctOStr ! diff --git a/driver-moab/main/prep_ocn_mod.F90 b/driver-moab/main/prep_ocn_mod.F90 index 44ec0c6a3c7a..84c5631bfecf 100644 --- a/driver-moab/main/prep_ocn_mod.F90 +++ b/driver-moab/main/prep_ocn_mod.F90 @@ -1,6 +1,6 @@ module prep_ocn_mod - use shr_kind_mod, only: r8 => SHR_KIND_R8 + use shr_kind_mod, only: R8 => SHR_KIND_R8 use shr_kind_mod, only: cs => SHR_KIND_CS use shr_kind_mod, only: cl => SHR_KIND_CL use shr_kind_mod, only: CX => shr_kind_CX, CXX => shr_kind_CXX @@ -137,7 +137,7 @@ module prep_ocn_mod integer , target :: x2oacc_ox_cnt ! x2oacc_ox: number of time samples accumulated ! accumulation variables for moab data - real (kind=r8) , allocatable, private, target :: x2oacc_om (:,:) ! Ocn import, ocn grid, cpl pes, moab array + real (kind=R8) , allocatable, private, target :: x2oacc_om (:,:) ! Ocn import, ocn grid, cpl pes, moab array integer , target :: x2oacc_om_cnt ! x2oacc_ox: number of time samples accumulated, in moab array integer :: arrSize_x2o_om ! this will be a module variable, size moabLocal_size * nof @@ -154,20 +154,20 @@ module prep_ocn_mod !================================================================================================ - real (kind=r8) , allocatable, private :: fractions_om (:,:) ! will retrieve the fractions from ocean, and use them + real (kind=R8) , allocatable, private :: fractions_om (:,:) ! will retrieve the fractions from ocean, and use them ! they were init with ! character(*),parameter :: fraclist_o = 'afrac:ifrac:ofrac:ifrad:ofrad' in moab, on the fractions - real (kind=r8) , allocatable, private :: x2o_om (:,:) - real (kind=r8) , allocatable, private :: a2x_om (:,:) - real (kind=r8) , allocatable, private :: i2x_om (:,:) - real (kind=r8) , allocatable, private :: r2x_om (:,:) - real (kind=r8) , allocatable, private :: xao_om (:,:) + real (kind=R8) , allocatable, private :: x2o_om (:,:) + real (kind=R8) , allocatable, private :: a2x_om (:,:) + real (kind=R8) , allocatable, private :: i2x_om (:,:) + real (kind=R8) , allocatable, private :: r2x_om (:,:) + real (kind=R8) , allocatable, private :: xao_om (:,:) ! this will be constructed first time, and be used to copy fields for shared indices ! between xao and x2o character(CXX) :: shared_fields_xao_x2o ! will need some array to hold the data for copying - real(r8) , allocatable, save :: shared_values(:) ! will be the size of shared indices * lsize + real(R8) , allocatable, save :: shared_values(:) ! will be the size of shared indices * lsize integer :: size_of_shared_values logical :: iamin_CPLALLICEID ! pe associated with CPLALLICEID @@ -203,7 +203,7 @@ subroutine prep_ocn_init(infodata, atm_c2_ocn, atm_c2_ice, ice_c2_ocn, rof_c2_oc use iMOAB, only: iMOAB_ComputeMeshIntersectionOnSphere, iMOAB_RegisterApplication, & iMOAB_WriteMesh, iMOAB_DefineTagStorage, iMOAB_ComputeCommGraph, iMOAB_ComputeScalarProjectionWeights, & iMOAB_MigrateMapMesh, iMOAB_WriteLocalMesh, iMOAB_GetMeshInfo, iMOAB_SetDoubleTagStorage, & - iMOAB_WriteMappingWeightsToFile + iMOAB_WriteMappingWeightsToFile, iMOAB_SetGhostLayers !--------------------------------------------------------------- ! Description ! Initialize module attribute vectors and all other non-mapping @@ -268,7 +268,8 @@ subroutine prep_ocn_init(infodata, atm_c2_ocn, atm_c2_ice, ice_c2_ocn, rof_c2_oc integer arrsize ! for setting the r2x fields on land to 0 integer ent_type ! for setting tags integer noflds ! used for number of fields in allocating moab accumulated array x2oacc_om - real (kind=r8) , allocatable :: tmparray (:) ! used to set the r2x fields to 0 + real (kind=R8) , allocatable :: tmparray (:) ! used to set the r2x fields to 0 + integer nghlay ! used to set the number of ghost layers, needed for bilinear map !--------------------------------------------------------------- @@ -426,6 +427,13 @@ subroutine prep_ocn_init(infodata, atm_c2_ocn, atm_c2_ice, ice_c2_ocn, rof_c2_oc ! next, let us compute the ATM and OCN data transfer if (.not. samegrid_ao) then ! not a data OCN model + ! for bilinear maps, we need to have a layer of ghosts on source + nghlay = 1 ! number of ghost layers + ierr = iMOAB_SetGhostLayers( mbaxid, nghlay ) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in setting the number of layers' + call shr_sys_abort(subname//' error in setting the number of layers') + endif ! first compute the overlap mesh between mbaxid (ATM) and mboxid (OCN) on coupler PEs ierr = iMOAB_ComputeMeshIntersectionOnSphere (mbaxid, mboxid, mbintxao) if (ierr .ne. 0) then @@ -786,7 +794,7 @@ subroutine prep_ocn_init(infodata, atm_c2_ocn, atm_c2_ice, ice_c2_ocn, rof_c2_oc arrsize = nrflds*mlsize allocate (tmparray(arrsize)) ! mlsize is the size of local land ! do we need to zero out others or just river ? - tmparray = 0._r8 + tmparray = 0._R8 ierr = iMOAB_SetDoubleTagStorage(mboxid, tagname, arrsize , ent_type, tmparray) if (ierr .ne. 0) then write(logunit,*) subname,' cant zero out r2x tags on ocn' @@ -1043,6 +1051,7 @@ subroutine prep_ocn_accum_avg_moab() use iMOAB, only : iMOAB_SetDoubleTagStorage, iMOAB_WriteMesh ! Local Variables integer :: ent_type, ierr + integer noflds, lsize ! used for restart case only? character(CXX) :: tagname character(*), parameter :: subname = '(prep_ocn_accum_avg_moab)' #ifdef MOABDEBUG @@ -1056,7 +1065,18 @@ subroutine prep_ocn_accum_avg_moab() x2oacc_om = 1./x2oacc_om_cnt * x2oacc_om end if + if (.not. allocated(x2o_om)) then + ! we could come here in the restart case; not sure why only for + ! the case ERS_Vmoab_T62_oQU120.CMPASO-NYF + lsize = size(x2oacc_om, 1) + noflds = size(x2oacc_om, 2) + allocate (x2o_om(lsize, noflds)) + arrSize_x2o_om = noflds * lsize + + endif + ! ***NOTE***THE FOLLOWING ACTUALLY MODIFIES x2o_om + x2o_om = x2oacc_om !call mct_avect_copy(x2oacc_ox(eoi), x2o_ox) ! modify the tags @@ -1095,7 +1115,7 @@ subroutine prep_ocn_mrg(infodata, fractions_ox, xao_ox, timer_mrg) ! ! Local Variables integer :: eii, ewi, egi, eoi, eai, eri, exi, efi, emi - real(r8) :: flux_epbalfact ! adjusted precip factor + real(R8) :: flux_epbalfact ! adjusted precip factor type(mct_avect), pointer :: x2o_ox integer :: cnt character(*), parameter :: subname = '(prep_ocn_mrg)' @@ -1177,7 +1197,7 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) !--------------------------------------------------------------- - real(r8) :: flux_epbalfact ! adjusted precip factor + real(R8) :: flux_epbalfact ! adjusted precip factor ! will build x2o_om , similar to x2o_ox ! no averages, just one ocn instance @@ -1187,11 +1207,11 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) integer :: kof,kif integer :: lsize, arrsize ! for double arrays integer , save :: noflds,naflds,niflds,nrflds,nxflds! ,ngflds,nwflds, no glacier or wave model - real(r8) :: ifrac,ifracr - real(r8) :: afrac,afracr - real(r8) :: frac_sum - real(r8) :: avsdr, anidr, avsdf, anidf ! albedos - real(r8) :: fswabsv, fswabsi ! sw + real(R8) :: ifrac,ifracr + real(R8) :: afrac,afracr + real(R8) :: frac_sum + real(R8) :: avsdr, anidr, avsdf, anidf ! albedos + real(R8) :: fswabsv, fswabsi ! sw character(CL),allocatable :: field_ocn(:) ! string converted to char character(CL),allocatable :: field_atm(:) ! string converted to char character(CL),allocatable :: field_ice(:) ! string converted to char @@ -1289,7 +1309,7 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) #endif #ifdef MOABCOMP character(CXX) :: mct_field - real(r8) :: difference + real(R8) :: difference type(mct_list) :: temp_list integer :: size_list, index_list type(mct_string) :: mctOStr ! @@ -1339,13 +1359,15 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) !nwflds = mct_aVect_nRattr(w2x_o) nxflds = mct_aVect_nRattr(xao_o) - !ngflds = mct_aVect_nRattr(g2x_o) - allocate(x2o_om (lsize, noflds)) - arrSize_x2o_om = lsize * noflds ! this willbe used to set/get x2o_om tags + if (.not. allocated(x2o_om)) then + !ngflds = mct_aVect_nRattr(g2x_o) + allocate(x2o_om (lsize, noflds)) + arrSize_x2o_om = lsize * noflds ! this willbe used to set/get x2o_om tags + endif allocate(a2x_om (lsize, naflds)) allocate(i2x_om (lsize, niflds)) allocate(r2x_om (lsize, nrflds)) - r2x_om = 0._r8 ! should we zero out all of them ? + r2x_om = 0._R8 ! should we zero out all of them ? allocate(xao_om (lsize, nxflds)) ! allocate fractions too ! use the fraclist fraclist_o = 'afrac:ifrac:ofrac:ifrad:ofrad' @@ -1769,7 +1791,7 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) ifrac = fractions_om(n,kif) ! fo_kif_ifrac(n) ! fractions_o%rAttr(kif,n) afrac = fractions_om(n,kof) ! fo_kof_ofrac(n) ! fractions_o%rAttr(kof,n) frac_sum = ifrac + afrac - if ((frac_sum) /= 0._r8) then + if ((frac_sum) /= 0._R8) then ifrac = ifrac / (frac_sum) afrac = afrac / (frac_sum) endif @@ -1777,7 +1799,7 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) ifracr = fractions_om(n,kir) ! fo_kir_ifrad(n) ! fractions_o%rAttr(kir,n) afracr = fractions_om(n,kor) ! fo_kor_ofrad(n) ! fractions_o%rAttr(kor,n) frac_sum = ifracr + afracr - if ((frac_sum) /= 0._r8) then + if ((frac_sum) /= 0._R8) then ifracr = ifracr / (frac_sum) afracr = afracr / (frac_sum) endif @@ -1913,7 +1935,7 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) ifrac = fractions_om(n,kif) !fo_kif_ifrac(n) ! fractions_o%rAttr(kif) afrac = fractions_om(n,kof) ! fo_kof_ofrac(n) ! fractions_o%rAttr(kof,n) frac_sum = ifrac + afrac - if ((frac_sum) /= 0._r8) then + if ((frac_sum) /= 0._R8) then ifrac = ifrac / (frac_sum) afrac = afrac / (frac_sum) endif @@ -2046,7 +2068,7 @@ subroutine prep_ocn_merge( flux_epbalfact, a2x_o, i2x_o, r2x_o, w2x_o, g2x_o, xa !----------------------------------------------------------------------- ! ! Arguments - real(r8) , intent(in) :: flux_epbalfact + real(R8) , intent(in) :: flux_epbalfact type(mct_aVect), intent(in) :: a2x_o type(mct_aVect), intent(in) :: i2x_o type(mct_aVect), intent(in) :: r2x_o @@ -2061,11 +2083,11 @@ subroutine prep_ocn_merge( flux_epbalfact, a2x_o, i2x_o, r2x_o, w2x_o, g2x_o, xa integer :: kof,kif integer :: lsize integer :: noflds,naflds,niflds,nrflds,nwflds,nxflds,ngflds - real(r8) :: ifrac,ifracr - real(r8) :: afrac,afracr - real(r8) :: frac_sum - real(r8) :: avsdr, anidr, avsdf, anidf ! albedos - real(r8) :: fswabsv, fswabsi ! sw + real(R8) :: ifrac,ifracr + real(R8) :: afrac,afracr + real(R8) :: frac_sum + real(R8) :: avsdr, anidr, avsdf, anidf ! albedos + real(R8) :: fswabsv, fswabsi ! sw character(CL),allocatable :: field_ocn(:) ! string converted to char character(CL),allocatable :: field_atm(:) ! string converted to char character(CL),allocatable :: field_ice(:) ! string converted to char @@ -2584,7 +2606,7 @@ subroutine prep_ocn_merge( flux_epbalfact, a2x_o, i2x_o, r2x_o, w2x_o, g2x_o, xa ifrac = fractions_o%rAttr(kif,n) afrac = fractions_o%rAttr(kof,n) frac_sum = ifrac + afrac - if ((frac_sum) /= 0._r8) then + if ((frac_sum) /= 0._R8) then ifrac = ifrac / (frac_sum) afrac = afrac / (frac_sum) endif @@ -2592,7 +2614,7 @@ subroutine prep_ocn_merge( flux_epbalfact, a2x_o, i2x_o, r2x_o, w2x_o, g2x_o, xa ifracr = fractions_o%rAttr(kir,n) afracr = fractions_o%rAttr(kor,n) frac_sum = ifracr + afracr - if ((frac_sum) /= 0._r8) then + if ((frac_sum) /= 0._R8) then ifracr = ifracr / (frac_sum) afracr = afracr / (frac_sum) endif @@ -2738,7 +2760,7 @@ subroutine prep_ocn_merge( flux_epbalfact, a2x_o, i2x_o, r2x_o, w2x_o, g2x_o, xa ifrac = fractions_o%rAttr(kif,n) afrac = fractions_o%rAttr(kof,n) frac_sum = ifrac + afrac - if ((frac_sum) /= 0._r8) then + if ((frac_sum) /= 0._R8) then ifrac = ifrac / (frac_sum) afrac = afrac / (frac_sum) endif @@ -3068,7 +3090,7 @@ function prep_ocn_get_mapper_Sw2o() prep_ocn_get_mapper_Sw2o => mapper_Sw2o end function prep_ocn_get_mapper_Sw2o function prep_ocn_get_x2oacc_om() - real(r8), DIMENSION(:, :), pointer :: prep_ocn_get_x2oacc_om + real(R8), DIMENSION(:, :), pointer :: prep_ocn_get_x2oacc_om prep_ocn_get_x2oacc_om => x2oacc_om end function prep_ocn_get_x2oacc_om function prep_ocn_get_x2oacc_om_cnt() diff --git a/driver-moab/main/prep_rof_mod.F90 b/driver-moab/main/prep_rof_mod.F90 index 59cc61c58890..2ab02f7bdeae 100644 --- a/driver-moab/main/prep_rof_mod.F90 +++ b/driver-moab/main/prep_rof_mod.F90 @@ -1,7 +1,7 @@ module prep_rof_mod #include "shr_assert.h" - use shr_kind_mod, only: r8 => SHR_KIND_R8 + use shr_kind_mod, only: R8 => SHR_KIND_R8 use shr_kind_mod, only: cs => SHR_KIND_CS use shr_kind_mod, only: cl => SHR_KIND_CL use shr_kind_mod, only: cxx => SHR_KIND_CXX @@ -30,6 +30,7 @@ module prep_rof_mod use component_type_mod, only: ocn ! used for context for projection towards ocean from rof ! use prep_lnd_mod, only: prep_lnd_get_mapper_Fr2l use map_lnd2rof_irrig_mod, only: map_lnd2rof_irrig + use seq_comm_mct, only: mb_rof_aream_computed ! signal use iso_c_binding #ifdef MOABCOMP @@ -112,22 +113,22 @@ module prep_rof_mod ! accumulation variables over moab fields character(CXX) :: sharedFieldsLndRof ! used in moab to define l2racc_lm - real (kind=r8) , allocatable, private, target :: l2racc_lm(:,:) ! lnd export, lnd grid, cpl pes - real (kind=r8) , allocatable, private :: l2x_lm2(:,:) ! basically l2x_lm, but in another copy, on rof module + real (kind=R8) , allocatable, private, target :: l2racc_lm(:,:) ! lnd export, lnd grid, cpl pes + real (kind=R8) , allocatable, private :: l2x_lm2(:,:) ! basically l2x_lm, but in another copy, on rof module integer , target :: l2racc_lm_cnt ! l2racc_lm: number of time samples accumulated integer :: nfields_sh_lr ! number of fields in sharedFieldsLndRof integer :: lsize_lm ! size of land in moab, local character(CXX) :: sharedFieldsAtmRof ! used in moab to define a2racc_am - real (kind=r8) , allocatable, private :: a2racc_am(:,:) ! atm export, atm grid, cpl pes - real (kind=r8) , allocatable, private :: a2x_am2(:,:) ! basically a2x_am, but in another copy, on rof module + real (kind=R8) , allocatable, private :: a2racc_am(:,:) ! atm export, atm grid, cpl pes + real (kind=R8) , allocatable, private :: a2x_am2(:,:) ! basically a2x_am, but in another copy, on rof module integer , target :: a2racc_am_cnt ! a2racc_am: number of time samples accumulated integer :: nfields_sh_ar ! number of fields in sharedFieldsAtmRof integer :: lsize_am ! size of atm in moab, local character(CXX) :: sharedFieldsOcnRof ! used in moab to define o2racc_om - real (kind=r8) , allocatable, private, target :: o2racc_om(:,:) ! ocn export, ocn grid, cpl pes - real (kind=r8) , allocatable, private :: o2r_om2(:,:) ! basically o2x_om, but in another copy, on rof module, only shared with rof + real (kind=R8) , allocatable, private, target :: o2racc_om(:,:) ! ocn export, ocn grid, cpl pes + real (kind=R8) , allocatable, private :: o2r_om2(:,:) ! basically o2x_om, but in another copy, on rof module, only shared with rof integer , target :: o2racc_om_cnt ! o2racc_om: number of time samples accumulated integer :: nfields_sh_or ! number of fields in sharedFieldsOcnRof integer :: lsize_om ! size of ocn in moab, local @@ -145,12 +146,12 @@ module prep_rof_mod logical :: samegrid_al ! samegrid atm and lnd ! moab stuff - real (kind=r8) , allocatable, private :: fractions_rm (:,:) ! will retrieve the fractions from rof, and use them + real (kind=R8) , allocatable, private :: fractions_rm (:,:) ! will retrieve the fractions from rof, and use them ! they were init with ! character(*),parameter :: fraclist_r = 'lfrac:lfrin:rfrac' in moab, on the fractions - real (kind=r8) , allocatable, private :: x2r_rm (:,:) ! result of merge - real (kind=r8) , allocatable, private :: a2x_rm (:,:) - real (kind=r8) , allocatable, private :: l2x_rm (:,:) + real (kind=R8) , allocatable, private :: x2r_rm (:,:) ! result of merge + real (kind=R8) , allocatable, private :: a2x_rm (:,:) + real (kind=R8) , allocatable, private :: l2x_rm (:,:) !================================================================================================ @@ -327,6 +328,7 @@ subroutine prep_rof_init(infodata, lnd_c2_rof, atm_c2_rof, ocn_c2_rof) write(logunit,*) subname,' error in defining tags for seq_flds_a2x_fields on rof cpl' call shr_sys_abort(subname//' ERROR in defining tags for seq_flds_a2x_fields on rof cpl') endif + call seq_comm_getData(CPLID ,mpigrp=mpigrp_CPLID) if (samegrid_lr) then ! the same mesh , lnd and rof use the same dofs, but restricted ! we do not compute intersection, so we will have to just send data from lnd to rof and viceversa, by GLOBAL_ID matching @@ -363,7 +365,6 @@ subroutine prep_rof_init(infodata, lnd_c2_rof, atm_c2_rof, ocn_c2_rof) ! we also need to compute the comm graph for the second hop, from the lnd on coupler to the ! lnd for the intx lnd-rof context (coverage) ! - call seq_comm_getData(CPLID ,mpigrp=mpigrp_CPLID) type1 = 3 ! land is FV now on coupler side type2 = 3; @@ -417,6 +418,7 @@ subroutine prep_rof_init(infodata, lnd_c2_rof, atm_c2_rof, ocn_c2_rof) fNoBubble, monotonicity, volumetric, fInverseDistanceMap, & noConserve, validate, & trim(dofnameS), trim(dofnameT) ) + mb_rof_aream_computed = .true. ! signal if (ierr .ne. 0) then write(logunit,*) subname,' error in computing lr weights ' call shr_sys_abort(subname//' ERROR in computing lr weights ') @@ -1223,7 +1225,7 @@ subroutine prep_rof_merge(l2x_r, a2x_r, fractions_r, x2r_r, cime_model,o2x_r) integer, save :: index_x2r_coszen_str integer, save :: index_frac - real(r8) :: frac + real(R8) :: frac character(CL) :: fracstr logical, save :: first_time = .true. logical, save :: flds_wiso_rof = .false. @@ -1529,7 +1531,7 @@ subroutine prep_rof_mrg_moab (infodata, cime_model) integer, save :: index_x2r_coszen_str integer, save :: index_frac - real(r8) :: frac + real(R8) :: frac character(CL) :: fracstr logical, save :: first_time = .true. logical, save :: flds_wiso_rof = .false. @@ -1548,7 +1550,7 @@ subroutine prep_rof_mrg_moab (infodata, cime_model) character*32 :: outfile, wopts, lnum #endif #ifdef MOABCOMP - real(r8) :: difference + real(R8) :: difference type(mct_list) :: temp_list integer :: size_list, index_list type(mct_string) :: mctOStr ! @@ -1995,7 +1997,7 @@ function prep_rof_get_o2racc_om_cnt() end function prep_rof_get_o2racc_om_cnt function prep_rof_get_o2racc_om() - real(r8), DIMENSION(:, :), pointer :: prep_rof_get_o2racc_om + real(R8), DIMENSION(:, :), pointer :: prep_rof_get_o2racc_om prep_rof_get_o2racc_om => o2racc_om end function prep_rof_get_o2racc_om @@ -2011,7 +2013,7 @@ function prep_rof_get_l2racc_lm_cnt() end function prep_rof_get_l2racc_lm_cnt function prep_rof_get_l2racc_lm() - real(r8), DIMENSION(:, :), pointer :: prep_rof_get_l2racc_lm + real(R8), DIMENSION(:, :), pointer :: prep_rof_get_l2racc_lm prep_rof_get_l2racc_lm => l2racc_lm end function prep_rof_get_l2racc_lm diff --git a/driver-moab/main/seq_frac_mct.F90 b/driver-moab/main/seq_frac_mct.F90 index a89fc0b9aa4f..031845a102ef 100644 --- a/driver-moab/main/seq_frac_mct.F90 +++ b/driver-moab/main/seq_frac_mct.F90 @@ -180,7 +180,7 @@ module seq_frac_mct use iMOAB, only : iMOAB_DefineTagStorage, iMOAB_SetDoubleTagStorage, & iMOAB_GetMeshInfo, iMOAB_SetDoubleTagStorageWithGid, iMOAB_WriteMesh, & - iMOAB_ApplyScalarProjectionWeights, iMOAB_SendElementTag, iMOAB_ReceiveElementTag, & + iMOAB_SendElementTag, iMOAB_ReceiveElementTag, & iMOAB_FreeSenderBuffers, iMOAB_GetDoubleTagStorage #ifdef MOABDEBUG use seq_comm_mct, only : num_moab_exports diff --git a/driver-moab/main/seq_rest_mod.F90 b/driver-moab/main/seq_rest_mod.F90 index ab40eab408a5..dafd4d056686 100644 --- a/driver-moab/main/seq_rest_mod.F90 +++ b/driver-moab/main/seq_rest_mod.F90 @@ -97,7 +97,7 @@ module seq_rest_mod public :: seq_rest_mb_write ! read cpl7_moab restart data #ifdef MOABDEBUG - public :: write_moab_state ! debug, write files + public :: write_moab_state ! debug, write files #endif ! !PUBLIC DATA MEMBERS: @@ -367,7 +367,7 @@ subroutine seq_rest_mb_read(rest_file, infodata, samegrid_al) use seq_comm_mct, only: mbaxid, mbixid, mboxid, mblxid, mbrxid, mbofxid ! coupler side instances use iMOAB, only: iMOAB_GetGlobalInfo use seq_comm_mct , only: num_moab_exports ! it is used only as a counter for moab h5m files - + implicit none character(*) , intent(in) :: rest_file ! restart file path/name @@ -379,7 +379,7 @@ subroutine seq_rest_mb_read(rest_file, infodata, samegrid_al) real(r8),allocatable :: ns(:) ! for reshaping diag data for restart file character(CXX) :: moab_rest_file - character(CXX) :: tagname + character(CXX) :: tagname integer (in), pointer :: o2racc_om_cnt ! replacement, moab version for o2racc_ox_cnt integer (in), pointer :: x2oacc_om_cnt ! replacement, moab version for x2oacc_ox_cnt @@ -392,7 +392,7 @@ subroutine seq_rest_mb_read(rest_file, infodata, samegrid_al) real(r8), dimension(:,:), pointer :: p_l2racc_lm character(len=*), parameter :: subname = "(seq_rest_mb_read) " - + !------------------------------------------------------------------------------- ! !------------------------------------------------------------------------------- @@ -517,7 +517,7 @@ subroutine seq_rest_mb_read(rest_file, infodata, samegrid_al) call seq_io_read(moab_rest_file, mboxid, 'fractions_ox', & 'afrac:ifrac:ofrac:ifrad:ofrad') ! fraclist_o = 'afrac:ifrac:ofrac:ifrad:ofrad' call seq_io_read(moab_rest_file, mboxid, 'o2x_ox', & - trim(seq_flds_o2x_fields)) + trim(seq_flds_o2x_fields)) tagname = trim(seq_flds_x2o_fields) x2oacc_om_cnt => prep_ocn_get_x2oacc_om_cnt() p_x2oacc_om => prep_ocn_get_x2oacc_om() @@ -525,7 +525,7 @@ subroutine seq_rest_mb_read(rest_file, infodata, samegrid_al) call seq_io_read (moab_rest_file, mboxid, 'x2oacc_ox', & trim(tagname), & matrix=p_x2oacc_om) - call seq_io_read(moab_rest_file, x2oacc_om_cnt, 'x2oacc_ox_cnt') + call seq_io_read(moab_rest_file, x2oacc_om_cnt, 'x2oacc_ox_cnt') ! tagname = trim(seq_flds_xao_fields)//C_NULL_CHAR ! arrsize = nxflds * lsize ! allocate (xao_om (lsize, nxflds)) ! ierr = iMOAB_GetDoubleTagStorage ( mbofxid, tagname, arrsize , ent_type, xao_om) @@ -548,7 +548,7 @@ subroutine seq_rest_mb_read(rest_file, infodata, samegrid_al) call seq_io_read(moab_rest_file, mbixid, 'fractions_ix', & 'afrac:ifrac:ofrac') ! fraclist_i = 'afrac:ifrac:ofrac' call seq_io_read(moab_rest_file, mbixid, 'i2x_ix', & - trim(seq_flds_i2x_fields) ) + trim(seq_flds_i2x_fields) ) ! gsmap => component_get_gsmap_cx(ice(1)) ! call seq_io_read(rest_file, gsmap, fractions_ix, 'fractions_ix') ! call seq_io_read(rest_file, ice, 'c2x', 'i2x_ix') @@ -557,7 +557,7 @@ subroutine seq_rest_mb_read(rest_file, infodata, samegrid_al) call seq_io_read(moab_rest_file, mbrxid, 'fractions_rx', & 'lfrac:lfrin:rfrac') ! fraclist_r = 'lfrac:lfrin:rfrac' call seq_io_read(moab_rest_file, mbrxid, 'r2x_rx', & - trim(seq_flds_r2x_fields) ) + trim(seq_flds_r2x_fields) ) ! gsmap => component_get_gsmap_cx(rof(1)) ! call seq_io_read(rest_file, gsmap, fractions_rx, 'fractions_rx') ! call seq_io_read(rest_file, rof, 'c2x', 'r2x_rx') @@ -799,6 +799,8 @@ subroutine seq_rest_write(EClock_d, seq_SyncClock, infodata, & call seq_io_write(rest_file,rvar,'seq_infodata_precip_fact',whead=whead,wdata=wdata) call seq_infodata_GetData(infodata,case_name=cvar) call seq_io_write(rest_file,trim(cvar),'seq_infodata_case_name',whead=whead,wdata=wdata) + call seq_infodata_GetData(infodata,rmean_rmv_ice_runoff=rvar) + call seq_io_write(rest_file,rvar,'seq_infodata_rmean_rmv_ice_runoff',whead=whead,wdata=wdata) call seq_timemgr_EClockGetData( EClock_d, start_ymd=ivar) call seq_io_write(rest_file,ivar,'seq_timemgr_start_ymd',whead=whead,wdata=wdata) @@ -1120,6 +1122,8 @@ subroutine seq_rest_mb_write(EClock_d, seq_SyncClock, infodata, & call seq_io_write(rest_file,rvar,'seq_infodata_precip_fact',whead=whead,wdata=wdata) call seq_infodata_GetData(infodata,case_name=cvar) call seq_io_write(rest_file,trim(cvar),'seq_infodata_case_name',whead=whead,wdata=wdata) + call seq_infodata_GetData(infodata,rmean_rmv_ice_runoff=rvar) + call seq_io_write(rest_file,rvar,'seq_infodata_rmean_rmv_ice_runoff',whead=whead,wdata=wdata) call seq_timemgr_EClockGetData( EClock_d, start_ymd=ivar) call seq_io_write(rest_file,ivar,'seq_timemgr_start_ymd',whead=whead,wdata=wdata) @@ -1169,15 +1173,15 @@ subroutine seq_rest_mb_write(EClock_d, seq_SyncClock, infodata, & ! nx for land will be from global nb atmosphere ierr = iMOAB_GetGlobalInfo(mbaxid, dummy, nx_lnd) ! max id for land will come from atm call seq_io_write(rest_file, mblxid, 'fractions_lx', & - 'afrac:lfrac:lfrin', & ! seq_frac_mod: character(*),parameter :: fraclist_l = 'afrac:lfrac:lfrin' - whead=whead, wdata=wdata, nx=nx_lnd) + 'afrac:lfrac:lfrin', & ! seq_frac_mod: character(*),parameter :: fraclist_l = 'afrac:lfrac:lfrin' + whead=whead, wdata=wdata, nx=nx_lnd) else call seq_io_write(rest_file, mblxid, 'fractions_lx', & - 'afrac:lfrac:lfrin', & ! seq_frac_mod: character(*),parameter :: fraclist_l = 'afrac:lfrac:lfrin' + 'afrac:lfrac:lfrin', & ! seq_frac_mod: character(*),parameter :: fraclist_l = 'afrac:lfrac:lfrin' whead=whead, wdata=wdata) endif ! call seq_io_write(rest_file, mblxid, 'fractions_lx', & - ! 'afrac:lfrac:lfrin', & ! seq_frac_mod: character(*),parameter :: fraclist_l = 'afrac:lfrac:lfrin' + ! 'afrac:lfrac:lfrin', & ! seq_frac_mod: character(*),parameter :: fraclist_l = 'afrac:lfrac:lfrin' ! whead=whead, wdata=wdata) ! gsmap => component_get_gsmap_cx(lnd(1)) ! call seq_io_write(rest_file, gsmap, fractions_lx, 'fractions_lx', & @@ -1192,7 +1196,7 @@ subroutine seq_rest_mb_write(EClock_d, seq_SyncClock, infodata, & ierr = iMOAB_GetGlobalInfo(mbaxid, dummy, nx_lnd) ! max id for land will come from atm call seq_io_write(rest_file, mblxid, 'l2racc_lx', & trim(tagname), & - whead=whead, wdata=wdata, matrix = p_l2racc_lm, nx=nx_lnd) + whead=whead, wdata=wdata, matrix = p_l2racc_lm, nx=nx_lnd) else call seq_io_write(rest_file, mblxid, 'l2racc_lx', & trim(tagname), & @@ -1247,11 +1251,11 @@ subroutine seq_rest_mb_write(EClock_d, seq_SyncClock, infodata, & if (ocn_present) then ! gsmap => component_get_gsmap_cx(ocn(1)) ! x2oacc_ox => prep_ocn_get_x2oacc_ox() - + call seq_io_write(rest_file, mboxid, 'fractions_ox', & 'afrac:ifrac:ofrac:ifrad:ofrad', & ! fraclist_o = 'afrac:ifrac:ofrac:ifrad:ofrad' whead=whead, wdata=wdata) - + call seq_io_write(rest_file, mboxid, 'o2x_ox', & trim(seq_flds_o2x_fields), & whead=whead, wdata=wdata) @@ -1293,7 +1297,7 @@ subroutine seq_rest_mb_write(EClock_d, seq_SyncClock, infodata, & whead=whead, wdata=wdata) call seq_io_write(rest_file, mbixid, 'i2x_ix', & trim(seq_flds_i2x_fields), & - whead=whead, wdata=wdata) + whead=whead, wdata=wdata) ! gsmap => component_get_gsmap_cx(ice(1)) ! call seq_io_write(rest_file, gsmap, fractions_ix, 'fractions_ix', & ! whead=whead, wdata=wdata) @@ -1348,10 +1352,10 @@ end subroutine seq_rest_mb_write !=============================================================================== #ifdef MOABDEBUG - subroutine write_moab_state ( before_reading ) ! debug, write files + subroutine write_moab_state ( before_reading ) ! debug, write files use seq_comm_mct, only: mbaxid, mbixid, mboxid, mblxid, mbrxid, mbofxid ! coupler side instances use seq_comm_mct, only: num_moab_exports - use iso_c_binding + use iso_c_binding use iMOAB, only: iMOAB_WriteMesh implicit none @@ -1414,7 +1418,7 @@ subroutine write_moab_state ( before_reading ) ! debug, write files endif endif - end subroutine write_moab_state + end subroutine write_moab_state #endif end module seq_rest_mod diff --git a/driver-moab/shr/seq_comm_mct.F90 b/driver-moab/shr/seq_comm_mct.F90 index 44d29e91ad2c..9c85df84d2b1 100644 --- a/driver-moab/shr/seq_comm_mct.F90 +++ b/driver-moab/shr/seq_comm_mct.F90 @@ -245,9 +245,9 @@ module seq_comm_mct integer, public :: mbintxar ! iMOAB id for intx mesh between atm and river integer, public :: mbintxlr ! iMOAB id for intx mesh between land and river integer, public :: mbintxrl ! iMOAB id for intx mesh between river and land - logical, public :: mb_land_mesh = .false. ! whether the land uses full FV mesh or not ; made true if domain mesh is read on comp land - + integer, public :: num_moab_exports ! iMOAB id for atm phys grid, on atm pes + logical, public :: mb_rof_aream_computed = .false. ! whether the aream for rof has been set or not !======================================================================= contains diff --git a/driver-moab/shr/seq_infodata_mod.F90 b/driver-moab/shr/seq_infodata_mod.F90 index 512adb853b63..d9e92f368ade 100644 --- a/driver-moab/shr/seq_infodata_mod.F90 +++ b/driver-moab/shr/seq_infodata_mod.F90 @@ -198,6 +198,7 @@ MODULE seq_infodata_mod logical :: ocn_prognostic ! does component model need input data from driver logical :: ocnrof_prognostic ! does component need rof data logical :: ocn_c2_glcshelf ! will ocn component send data for ice shelf fluxes in driver + logical :: ocn_c2_glctf ! will ocn component send data for thermal forcing in driver logical :: ice_present ! does component model exist logical :: ice_prognostic ! does component model need input data from driver logical :: iceberg_prognostic ! does the ice model support icebergs @@ -251,7 +252,8 @@ MODULE seq_infodata_mod integer(SHR_KIND_IN) :: iac_phase ! iac phase logical :: atm_aero ! atmosphere aerosols logical :: glc_g2lupdate ! update glc2lnd fields in lnd model - real(shr_kind_r8) :: max_cplstep_time ! abort if cplstep time exceeds this value + real(SHR_KIND_R8) :: max_cplstep_time ! abort if cplstep time exceeds this value + real(SHR_KIND_R8) :: rmean_rmv_ice_runoff ! running mean of removed Antarctic ice runoff !--- set from restart file --- character(SHR_KIND_CL) :: rest_case_name ! Short case identification !--- set by driver and may be time varying @@ -759,10 +761,11 @@ SUBROUTINE seq_infodata_Init( infodata, nmlfile, ID, pioid, cpl_tag) infodata%atm_prognostic = .false. infodata%lnd_prognostic = .false. infodata%rof_prognostic = .false. - infodata%rofocn_prognostic = .false. + infodata%rofocn_prognostic = .false. infodata%ocn_prognostic = .false. infodata%ocnrof_prognostic = .false. infodata%ocn_c2_glcshelf = .false. + infodata%ocn_c2_glctf = .false. infodata%ice_prognostic = .false. infodata%glc_prognostic = .false. ! It's safest to assume glc_coupled_fluxes = .true. if it's not set elsewhere, @@ -795,8 +798,8 @@ SUBROUTINE seq_infodata_Init( infodata, nmlfile, ID, pioid, cpl_tag) infodata%lnd_domain = 'none' infodata%rof_mesh = 'none' infodata%rof_domain = 'none' - infodata%ocn_domain = 'none' ! will be used for ocean data models only; will be used as a signal - infodata%ice_domain = 'none' ! will be used for ice data models only; will be used as a signal + infodata%ocn_domain = 'none' ! will be used for ocean data models only; will be used as a signal + infodata%ice_domain = 'none' ! will be used for ice data models only; will be used as a signal infodata%atm_mesh = 'none' ! will be used for atmosphere data models only; will be used as a signal ! not sure if it exists always actually @@ -813,6 +816,7 @@ SUBROUTINE seq_infodata_Init( infodata, nmlfile, ID, pioid, cpl_tag) infodata%atm_aero = .false. infodata%glc_g2lupdate = .false. infodata%glc_valid_input = .true. + infodata%rmean_rmv_ice_runoff = -1.0_SHR_KIND_R8 infodata%max_cplstep_time = max_cplstep_time infodata%model_doi_url = model_doi_url @@ -912,11 +916,13 @@ SUBROUTINE seq_infodata_Init( infodata, nmlfile, ID, pioid, cpl_tag) call seq_io_read(infodata%restart_file,pioid,infodata%nextsw_cday ,'seq_infodata_nextsw_cday') call seq_io_read(infodata%restart_file,pioid,infodata%precip_fact ,'seq_infodata_precip_fact') call seq_io_read(infodata%restart_file,pioid,infodata%rest_case_name,'seq_infodata_case_name') + call seq_io_read(infodata%restart_file,pioid,infodata%rmean_rmv_ice_runoff ,'seq_infodata_rmean_rmv_ice_runoff') endif !--- Send from CPLID ROOT to GLOBALID ROOT, use bcast as surrogate call shr_mpi_bcast(infodata%nextsw_cday,mpicom,pebcast=seq_comm_gloroot(CPLID)) call shr_mpi_bcast(infodata%precip_fact,mpicom,pebcast=seq_comm_gloroot(CPLID)) call shr_mpi_bcast(infodata%rest_case_name,mpicom,pebcast=seq_comm_gloroot(CPLID)) + call shr_mpi_bcast(infodata%rmean_rmv_ice_runoff,mpicom,pebcast=seq_comm_gloroot(CPLID)) endif if (seq_comm_iamroot(ID)) then @@ -1009,7 +1015,8 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ atm_present, atm_prognostic, & lnd_present, lnd_prognostic, & rof_present, rof_prognostic, rofocn_prognostic, & - ocn_present, ocn_prognostic, ocnrof_prognostic, ocn_c2_glcshelf, & + ocn_present, ocn_prognostic, ocnrof_prognostic, & + ocn_c2_glcshelf, ocn_c2_glctf, & ice_present, ice_prognostic, & glc_present, glc_prognostic, & iac_present, iac_prognostic, & @@ -1047,7 +1054,7 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ reprosum_use_ddpdd, reprosum_allow_infnan, & reprosum_diffmax, reprosum_recompute, & mct_usealltoall, mct_usevector, max_cplstep_time, model_doi_url, & - glc_valid_input, nlmaps_verbosity) + glc_valid_input, nlmaps_verbosity, rmean_rmv_ice_runoff) implicit none @@ -1183,6 +1190,7 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ logical, optional, intent(OUT) :: ocn_prognostic logical, optional, intent(OUT) :: ocnrof_prognostic logical, optional, intent(OUT) :: ocn_c2_glcshelf + logical, optional, intent(OUT) :: ocn_c2_glctf logical, optional, intent(OUT) :: ice_present logical, optional, intent(OUT) :: ice_prognostic logical, optional, intent(OUT) :: iceberg_prognostic @@ -1238,6 +1246,7 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ real(shr_kind_r8), optional, intent(out) :: max_cplstep_time character(SHR_KIND_CL), optional, intent(OUT) :: model_doi_url logical, optional, intent(OUT) :: glc_valid_input + real(SHR_KIND_R8), optional, intent(OUT) :: rmean_rmv_ice_runoff ! running mean of removed Antarctic ice runoff !----- local ----- character(len=*), parameter :: subname = '(seq_infodata_GetData_explicit) ' @@ -1374,6 +1383,7 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ if ( present(ocn_prognostic) ) ocn_prognostic = infodata%ocn_prognostic if ( present(ocnrof_prognostic) ) ocnrof_prognostic = infodata%ocnrof_prognostic if ( present(ocn_c2_glcshelf) ) ocn_c2_glcshelf = infodata%ocn_c2_glcshelf + if ( present(ocn_c2_glctf) ) ocn_c2_glctf = infodata%ocn_c2_glctf if ( present(ice_present) ) ice_present = infodata%ice_present if ( present(ice_prognostic) ) ice_prognostic = infodata%ice_prognostic if ( present(iceberg_prognostic)) iceberg_prognostic = infodata%iceberg_prognostic @@ -1442,6 +1452,7 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ if ( present(model_doi_url) ) model_doi_url = infodata%model_doi_url if ( present(glc_valid_input)) glc_valid_input = infodata%glc_valid_input + if ( present(rmean_rmv_ice_runoff) ) rmean_rmv_ice_runoff = infodata%rmean_rmv_ice_runoff END SUBROUTINE seq_infodata_GetData_explicit @@ -1572,7 +1583,8 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ atm_present, atm_prognostic, & lnd_present, lnd_prognostic, & rof_present, rof_prognostic, rofocn_prognostic, & - ocn_present, ocn_prognostic, ocnrof_prognostic, ocn_c2_glcshelf, & + ocn_present, ocn_prognostic, ocnrof_prognostic, & + ocn_c2_glcshelf, ocn_c2_glctf, & ice_present, ice_prognostic, & glc_present, glc_prognostic, & glc_coupled_fluxes, & @@ -1610,7 +1622,8 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ eps_aarea, eps_omask, eps_ogrid, eps_oarea, & reprosum_use_ddpdd, reprosum_allow_infnan, & reprosum_diffmax, reprosum_recompute, & - mct_usealltoall, mct_usevector, glc_valid_input, nlmaps_verbosity) + mct_usealltoall, mct_usevector, glc_valid_input, nlmaps_verbosity, & + rmean_rmv_ice_runoff) implicit none @@ -1746,6 +1759,7 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ logical, optional, intent(IN) :: ocn_prognostic logical, optional, intent(IN) :: ocnrof_prognostic logical, optional, intent(IN) :: ocn_c2_glcshelf + logical, optional, intent(IN) :: ocn_c2_glctf logical, optional, intent(IN) :: ice_present logical, optional, intent(IN) :: ice_prognostic logical, optional, intent(IN) :: iceberg_prognostic @@ -1798,6 +1812,7 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ logical, optional, intent(IN) :: atm_aero ! atm aerosols logical, optional, intent(IN) :: glc_g2lupdate ! update glc2lnd fields in lnd model logical, optional, intent(IN) :: glc_valid_input + real(SHR_KIND_R8), optional, intent(IN) :: rmean_rmv_ice_runoff ! running mean of removed Antarctic ice runoff !EOP @@ -1936,6 +1951,7 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ if ( present(ocn_prognostic) ) infodata%ocn_prognostic = ocn_prognostic if ( present(ocnrof_prognostic)) infodata%ocnrof_prognostic = ocnrof_prognostic if ( present(ocn_c2_glcshelf)) infodata%ocn_c2_glcshelf = ocn_c2_glcshelf + if ( present(ocn_c2_glctf)) infodata%ocn_c2_glctf = ocn_c2_glctf if ( present(ice_present) ) infodata%ice_present = ice_present if ( present(ice_prognostic) ) infodata%ice_prognostic = ice_prognostic if ( present(iceberg_prognostic)) infodata%iceberg_prognostic = iceberg_prognostic @@ -1988,6 +2004,7 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ if ( present(atm_aero) ) infodata%atm_aero = atm_aero if ( present(glc_g2lupdate) ) infodata%glc_g2lupdate = glc_g2lupdate if ( present(glc_valid_input) ) infodata%glc_valid_input = glc_valid_input + if ( present(rmean_rmv_ice_runoff) ) infodata%rmean_rmv_ice_runoff = rmean_rmv_ice_runoff END SUBROUTINE seq_infodata_PutData_explicit @@ -2249,6 +2266,7 @@ subroutine seq_infodata_bcast(infodata,mpicom) call shr_mpi_bcast(infodata%ocn_prognostic, mpicom) call shr_mpi_bcast(infodata%ocnrof_prognostic, mpicom) call shr_mpi_bcast(infodata%ocn_c2_glcshelf, mpicom) + call shr_mpi_bcast(infodata%ocn_c2_glctf, mpicom) call shr_mpi_bcast(infodata%ice_present, mpicom) call shr_mpi_bcast(infodata%ice_prognostic, mpicom) call shr_mpi_bcast(infodata%iceberg_prognostic, mpicom) @@ -2302,6 +2320,7 @@ subroutine seq_infodata_bcast(infodata,mpicom) call shr_mpi_bcast(infodata%glc_valid_input, mpicom) call shr_mpi_bcast(infodata%model_doi_url, mpicom) call shr_mpi_bcast(infodata%constant_zenith_deg, mpicom) + call shr_mpi_bcast(infodata%rmean_rmv_ice_runoff, mpicom) end subroutine seq_infodata_bcast @@ -2544,6 +2563,7 @@ subroutine seq_infodata_Exchange(infodata,ID,type) call shr_mpi_bcast(infodata%ocn_prognostic, mpicom, pebcast=cmppe) call shr_mpi_bcast(infodata%ocnrof_prognostic, mpicom, pebcast=cmppe) call shr_mpi_bcast(infodata%ocn_c2_glcshelf, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%ocn_c2_glctf, mpicom, pebcast=cmppe) call shr_mpi_bcast(infodata%ocn_nx, mpicom, pebcast=cmppe) call shr_mpi_bcast(infodata%ocn_ny, mpicom, pebcast=cmppe) call shr_mpi_bcast(infodata%ocn_domain, mpicom, pebcast=cmppe) @@ -2622,6 +2642,7 @@ subroutine seq_infodata_Exchange(infodata,ID,type) call shr_mpi_bcast(infodata%ocn_prognostic, mpicom, pebcast=cplpe) call shr_mpi_bcast(infodata%ocnrof_prognostic, mpicom, pebcast=cplpe) call shr_mpi_bcast(infodata%ocn_c2_glcshelf, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%ocn_c2_glctf, mpicom, pebcast=cplpe) call shr_mpi_bcast(infodata%ice_present, mpicom, pebcast=cplpe) call shr_mpi_bcast(infodata%ice_prognostic, mpicom, pebcast=cplpe) call shr_mpi_bcast(infodata%iceberg_prognostic, mpicom, pebcast=cplpe) @@ -2648,6 +2669,7 @@ subroutine seq_infodata_Exchange(infodata,ID,type) if (ocn2cplr) then call shr_mpi_bcast(infodata%precip_fact, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%rmean_rmv_ice_runoff, mpicom, pebcast=cmppe) endif if (cpl2r) then @@ -2655,6 +2677,7 @@ subroutine seq_infodata_Exchange(infodata,ID,type) call shr_mpi_bcast(infodata%precip_fact, mpicom, pebcast=cplpe) call shr_mpi_bcast(infodata%glc_g2lupdate, mpicom, pebcast=cplpe) call shr_mpi_bcast(infodata%glc_valid_input, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%rmean_rmv_ice_runoff, mpicom, pebcast=cplpe) endif end subroutine seq_infodata_Exchange @@ -2972,6 +2995,7 @@ SUBROUTINE seq_infodata_print( infodata ) write(logunit,F0L) subname,'ocn_prognostic = ', infodata%ocn_prognostic write(logunit,F0L) subname,'ocnrof_prognostic = ', infodata%ocnrof_prognostic write(logunit,F0L) subname,'ocn_c2_glcshelf = ', infodata%ocn_c2_glcshelf + write(logunit,F0L) subname,'ocn_c2_glctf = ', infodata%ocn_c2_glctf write(logunit,F0L) subname,'ice_present = ', infodata%ice_present write(logunit,F0L) subname,'ice_prognostic = ', infodata%ice_prognostic write(logunit,F0L) subname,'iceberg_prognostic = ', infodata%iceberg_prognostic @@ -3025,6 +3049,7 @@ SUBROUTINE seq_infodata_print( infodata ) write(logunit,F0S) subname,'iac_phase = ', infodata%iac_phase write(logunit,F0L) subname,'glc_g2lupdate = ', infodata%glc_g2lupdate + write(logunit,F0R) subname,'rmean_rmv_ice_runoff = ', infodata%rmean_rmv_ice_runoff ! endif END SUBROUTINE seq_infodata_print diff --git a/externals/mam4xx b/externals/mam4xx index 2eee1988b1e6..f3f215185ca5 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 2eee1988b1e6488c904e0a28564bdcadfa190d34 +Subproject commit f3f215185ca59cd765704e763c499b484791fc62 diff --git a/externals/scorpio b/externals/scorpio index 804f87c48fc9..59601f625e34 160000 --- a/externals/scorpio +++ b/externals/scorpio @@ -1 +1 @@ -Subproject commit 804f87c48fc9013009307b806d1aff2afa3a0d7c +Subproject commit 59601f625e34019de0e1d3411d07bdc9e70d0054 diff --git a/share/build/buildlib.spio b/share/build/buildlib.spio index d17627c4323d..064fa751e4ae 100755 --- a/share/build/buildlib.spio +++ b/share/build/buildlib.spio @@ -78,6 +78,8 @@ def buildlib(bldroot, installpath, case): # ADIOS2_ROOT. This is a workaround if "ADIOS2_ROOT" in os.environ: os.environ["ADIOS2_DIR"] = os.environ["ADIOS2_ROOT"] + if "BLOSC2_ROOT" in os.environ: + os.environ["Blosc2_DIR"] = os.environ["BLOSC2_ROOT"] # If variable PIO_VERSION_MAJOR is defined in the environment then # we assume that PIO is installed on the system @@ -105,10 +107,14 @@ def buildlib(bldroot, installpath, case): else: cmake_opts += f"-DGENF90_PATH={scorpio_src_dir}/src/genf90 " + adiosc_found = False if "ADIOS2_ROOT" in os.environ: cmake_opts += "-DWITH_ADIOS2:BOOL=ON " if "FROM_CREATE_TEST" in os.environ and os.environ["FROM_CREATE_TEST"] == "True": cmake_opts += "-DADIOS_BP2NC_TEST:BOOL=ON " + if "BLOSC2_ROOT" in os.environ: + cmake_opts += "-DADIOS_USE_COMPRESSION:BOOL=ON " + adiosc_found = True if debug: cmake_opts += "-DPIO_ENABLE_LOGGING=ON " @@ -281,6 +287,8 @@ def buildlib(bldroot, installpath, case): valid_values += ",netcdf4p,netcdf4c" if adios_found: valid_values += ",adios" + if adiosc_found: + valid_values += ",adiosc" if hdf5_found: valid_values += ",hdf5" diff --git a/share/pacer/Pacer.cpp b/share/pacer/Pacer.cpp new file mode 100644 index 000000000000..ac1ad329b542 --- /dev/null +++ b/share/pacer/Pacer.cpp @@ -0,0 +1,211 @@ +//===-- Pacer.cpp - Timers for E3SM --*- C++ -*-===// +// +// \file +// \brief Timer infrastructure for E3SM C++ components +// +// +////===------------------------------------------===// + +#include "Pacer.h" +#include +#include +#include + +#define PACER_CHECK_INIT() {\ + if (!IsInitialized) { \ + std::cerr << "[ERROR] Pacer: Not initialized." << std::endl; \ + return false; \ + } \ +} + +#define PACER_CHECK_ERROR(x) {\ + if ( (x) != 0 ) { \ + std::cerr << "[ERROR] Pacer: Failure calling GPTL function: " << #x << std::endl; \ + return false; \ + } \ +} + +/// Helper function to check if GPTL is initialized +/// Function declaration is missing in gptl.h +/// Hence the declaration here +extern "C" { + extern int GPTLis_initialized(void); +} + +/// Check if Pacer is initialized +/// Returns true if initialized +inline bool Pacer::isInitialized(void){ + if (!IsInitialized) { + std::cerr << "[ERROR] Pacer: Not initialized." << std::endl; + return false; + } + return true; +} + +/// Initialize Pacer timing +/// InComm: overall MPI communicator used by application. +/// InMode: Pacer standalone (default) or within CIME +bool Pacer::initialize(MPI_Comm InComm, PacerModeType InMode /* = PACER_STANDALONE */) { + + int errCode; + + PacerMode = InMode; + + // Check if already initialized and return + if (IsInitialized) + return true; + + // Duplicate comm and get MPI rank for both standalone and integrated modes + if (MPI_Comm_dup(InComm, &InternalComm) != MPI_SUCCESS) + std::cerr << "Pacer: Error duplicating MPI communicator" << std::endl; + MPI_Comm_rank(InternalComm, &MyRank); + + if (PacerMode == PACER_STANDALONE ) { + // GPTL set default options + PACER_CHECK_ERROR(GPTLsetoption(GPTLdepthlimit, 20)); + PACER_CHECK_ERROR(GPTLsetoption(GPTLdopr_quotes, 1)); + PACER_CHECK_ERROR(GPTLsetoption(GPTLprofile_ovhd, 1)); + // GPTL default is set to 52 + // Presently setting to 64 + PACER_CHECK_ERROR(GPTLsetoption(GPTLmaxthreads, 64)); + + PACER_CHECK_ERROR(GPTLsetutr(GPTLmpiwtime)); + + errCode = GPTLinitialize(); + + if (errCode) { + std::cerr << "[ERROR] Pacer: Unable to initialize GPTL." << std::endl; + return false; + } + else + IsInitialized = true; + } + else if (PacerMode == PACER_INTEGRATED) { + // GPTL is assumed to be initialized by E3SM driver in coupled modeling context + + // Check if GPTL is initialized + errCode = GPTLis_initialized(); + + if (errCode == true) { + IsInitialized = true; + } + else { + IsInitialized = false; + std::cerr << "[ERROR] Pacer: GPTL is not initialized in PACER_INTEGRATED mode." << std::endl; + return false; + } + } + + return true; +} + +/// Start the time named TimerName +bool Pacer::start(const std::string &TimerName) +{ + PACER_CHECK_INIT(); + + PACER_CHECK_ERROR(GPTLstart(TimerName.c_str())); + + auto it = OpenTimers.find(TimerName); + if (it != OpenTimers.end() ) + OpenTimers[TimerName]++; + else + OpenTimers[TimerName] = 1; + return true; +} + +/// Stop the time named TimerName +/// Issues warning if timer hasn't been started yet +bool Pacer::stop(const std::string &TimerName) +{ + PACER_CHECK_INIT(); + + auto it = OpenTimers.find(TimerName); + + if (it != OpenTimers.end() ) { + PACER_CHECK_ERROR(GPTLstop(TimerName.c_str())); + + if ( OpenTimers[TimerName] == 1 ) + OpenTimers.erase(TimerName); + else + OpenTimers[TimerName]--; + } + else { + std::cerr << "[WARNING] Pacer: Trying to stop timer: \"" + << TimerName << "\" before starting it." << std::endl; + + return false; + } + + return true; +} + +/// Sets named prefix for all subsequent timers +bool Pacer::setPrefix(const std::string &Prefix) +{ + PACER_CHECK_INIT(); + + PACER_CHECK_ERROR(GPTLprefix_set(Prefix.c_str())); + + return true; +} + +/// Unsets prefix for all subsequent timers +bool Pacer::unsetPrefix() +{ + PACER_CHECK_INIT(); + + PACER_CHECK_ERROR(GPTLprefix_unset()); + + return true; +} + +/// Prints timing statistics and global summary files +/// Output Files: TimerFilePrefix.timing. +/// TimerFilePrefix.summary +/// PrintAllRanks: flag to control if per rank timing files are printed +bool Pacer::print(const std::string &TimerFilePrefix, bool PrintAllRanks /*= = false */) +{ + PACER_CHECK_INIT(); + + std::string TimerFileName = TimerFilePrefix + ".timing." + std::to_string(MyRank); + std::string SummaryFileName = TimerFilePrefix + ".summary"; + + PACER_CHECK_ERROR(GPTLpr_summary_file(InternalComm, SummaryFileName.c_str())); + + if ( PrintAllRanks == false ) { + if (MyRank == 0) { + PACER_CHECK_ERROR(GPTLpr_file(TimerFileName.c_str())); + } + } + else + PACER_CHECK_ERROR(GPTLpr_file(TimerFileName.c_str())); + + return true; +} + +/// Cleans up Pacer +/// Issues warning if any timers are still open +bool Pacer::finalize() +{ + PACER_CHECK_INIT(); + + if ( PacerMode == PACER_STANDALONE ) + PACER_CHECK_ERROR(GPTLfinalize()); + + if ( (MyRank == 0) && ( OpenTimers.size() > 0) ){ + std::cerr << "[WARNING] Pacer: Following " << OpenTimers.size() << " timer(s) is/are still open." << std::endl; + for (auto i = OpenTimers.begin(); i != OpenTimers.end(); i++) + std::cerr << '\t' << i->first << std::endl; + } + OpenTimers.clear(); + + // Clear Pacer state and free communicator + IsInitialized = false; + MPI_Comm_free(&InternalComm); + + return true; +} + + +//===----------------------------------------------------------------------------===// diff --git a/share/pacer/Pacer.h b/share/pacer/Pacer.h new file mode 100644 index 000000000000..3c64f001fa87 --- /dev/null +++ b/share/pacer/Pacer.h @@ -0,0 +1,75 @@ +//===-- Pacer.h - Pacer timing interface -------------*- C++ +//-*-===// +// +// \file +// \brief Provides timer functionality for E3SM +// +// The Pacer class provides an interface to timers for +// E3SM components. +// +//===--------------------------------------------------===// +#ifndef PACER_H +#define PACER_H + +#include +#include +#include +#include + +namespace Pacer { + + /// Flag to determine if the timing infrastructure is initialized + static bool IsInitialized = false; + + /// MPI communicator used within Pacer + static MPI_Comm InternalComm; + + /// MPI rank of process + static int MyRank; + + /// hash table of open timers with count (for multiple parents) + static std::unordered_map OpenTimers; + + enum PacerModeType { PACER_STANDALONE, PACER_INTEGRATED }; + + /// Pacer Mode: standalone or within CIME + static PacerModeType PacerMode; + + /// Initialize Pacer timing + /// InComm: overall MPI communicator used by application. + /// InMode: Pacer Mode: standalone (default) or within CIME + bool initialize(MPI_Comm InComm, PacerModeType InMode = PACER_STANDALONE); + + /// Check if Pacer is initialized + /// Returns true if initialized + bool isInitialized(void); + + /// IntegratedMode: Pacer standalone (default:false) or within CIME (true) + // bool initialize(MPI_Comm InComm, bool IntegratedMode = false); + + /// Start the time named TimerName + bool start(const std::string &TimerName); + + /// Stop the time named TimerName + /// Issues warning if timer hasn't been started yet + bool stop(const std::string &TimerName); + + /// Sets named prefix for all subsequent timers + bool setPrefix(const std::string &Prefix); + + /// Unsets prefix for all subsequent timers + bool unsetPrefix(); + + /// Prints timing statistics and global summary files + /// Output Files: TimerFilePrefix.timing. + /// TimerFilePrefix.summary + /// PrintAllRanks: flag to control if per rank timing files are printed + bool print(const std::string &TimerFilePrefix, bool PrintAllRanks = false); + + /// Cleans up Pacer + /// Issues warning if any timers are still open + bool finalize(); + +}; + +#endif diff --git a/share/pacer/TestPacer.cpp b/share/pacer/TestPacer.cpp new file mode 100644 index 000000000000..96f297862324 --- /dev/null +++ b/share/pacer/TestPacer.cpp @@ -0,0 +1,64 @@ +////===-- TestPacer.cpp - Simple test for Pacer --*- C++ -*-===// +// +// \file +// \brief Simple example illustrating Pacer API usage. +// +// This test exercises basic timer functionality +// with the Pacer API. +// +// This test program should create two files: +// test.timing.0 and test.summary +// It is also expected to issue couple of warnings +// to illustrate likely scenarios where a timer is +// not started/stopped properly. +// +////===-----------------------------------------------------===// + +#include +#include +#include "Pacer.h" + +int main(int argc, char **argv){ + + int err; + int myrank; + + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &myrank); + + Pacer::initialize(MPI_COMM_WORLD); + // Second argument is optional (default is Pacer::PACER_STANDALONE) + // Pacer::initialize(MPI_COMM_WORLD, Pacer::PACER_STANDALONE); + + Pacer::setPrefix("Omega:"); + + Pacer::start("run_loop"); + + float tmp = 1; + + for (int i = 1; i <= 10000; i++){ + tmp *= i; + } + + Pacer::stop("run_loop"); + + Pacer::unsetPrefix(); + + // illustrating situation where attempt to stop timer before starting + // will print a warning + Pacer::stop("final"); + + // as well as dangling timer which is never stopped explicitly + // will print a warning at the end during finalize + Pacer::start("final"); + + + // print: First argument is the prefix used for timing output file names + // Second argument (optional) controls if timing output should be from all ranks + // default is false, only rank 0 writes timing output + Pacer::print("test"); + // Pacer::print("test", true); + + Pacer::finalize(); +} + diff --git a/share/util/shr_pio_mod.F90 b/share/util/shr_pio_mod.F90 index d500cbc680a2..11cefd725b54 100644 --- a/share/util/shr_pio_mod.F90 +++ b/share/util/shr_pio_mod.F90 @@ -679,6 +679,8 @@ subroutine shr_pio_getiotypefromname(typename, iotype, defaulttype) #ifndef PIO1 else if ( typename .eq. 'ADIOS') then iotype = pio_iotype_adios + else if ( typename .eq. 'ADIOSC') then + iotype = pio_iotype_adiosc else if ( typename .eq. 'HDF5') then iotype = pio_iotype_hdf5 #endif diff --git a/share/util/shr_scam_mod.F90 b/share/util/shr_scam_mod.F90 index dd82ab5e1714..2fb92f769cc0 100644 --- a/share/util/shr_scam_mod.F90 +++ b/share/util/shr_scam_mod.F90 @@ -647,11 +647,13 @@ subroutine shr_scam_checkSurface(scmlon, scmlat, scm_multcols, scm_nx, scm_ny, & real(r8) :: sst_constant_value character(len=CL) :: restfilm = 'unset' character(len=CL) :: restfils = 'unset' + real(r8) :: RSO_fixed_MLD + real(r8) :: RSO_relax_tau integer(IN) :: nfrac logical :: force_prognostic_true = .false. namelist /dom_inparm/ sstcyc, nrevsn, rest_pfile, bndtvs, focndomain namelist / docn_nml / decomp, sst_constant_value, force_prognostic_true, & - restfilm, restfils + restfilm, restfils, RSO_fixed_MLD, RSO_relax_tau !------------------------------------------------------------------------------- ! Notes: