Merge branch 'development' into fix_omp_diag

.github/workflows/clang-tidy.yml

@@ -0,0 +1,36 @@
+name: "clang-tidy"
+
+on: [pull_request]
+
+concurrency:
+  group: ${{ github.ref }}-${{ github.head_ref }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  clang_tidy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Get submodules
+        run: |
+          git submodule update --init
+          cd external/Microphysics
+          git fetch; git checkout development
+          echo "MICROPHYSICS_HOME=$(pwd)" >> $GITHUB_ENV
+          cd ../amrex
+          git fetch; git checkout development
+          echo "AMREX_HOME=$(pwd)" >> $GITHUB_ENV
+          cd ../..
+
+      - name: Install dependencies
+        run: |
+          .github/workflows/dependencies_clang-tidy-apt-llvm.sh 17
+
+      - name: Compile wdconvect
+        run: |
+          echo $AMREX_HOME
+          echo $MICROPHYSICS_HOME
+          cd Exec/science/wdconvect
+          make USE_MPI=FALSE USE_OMP=FALSE USE_CLANG_TIDY=TRUE CLANG_TIDY=clang-tidy-17 CLANG_TIDY_WARN_ERROR=TRUE -j 4
+

.github/workflows/dependencies_clang-tidy-apt-llvm.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+# `man apt.conf`:
+#   Number of retries to perform. If this is non-zero APT will retry
+#   failed files the given number of times.
+echo 'Acquire::Retries "3";' | sudo tee /etc/apt/apt.conf.d/80-retries
+
+if [[ ! -f /etc/apt/trusted.gpg.d/apt.llvm.org.asc ]]; then
+    wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
+fi
+
+source /etc/os-release # set UBUNTU_CODENAME
+
+sudo add-apt-repository "deb http://apt.llvm.org/${UBUNTU_CODENAME}/ llvm-toolchain-${UBUNTU_CODENAME} main"
+sudo add-apt-repository "deb http://apt.llvm.org/${UBUNTU_CODENAME}/ llvm-toolchain-${UBUNTU_CODENAME}-$1 main"
+
+sudo apt-get update
+
+sudo apt-get install -y --no-install-recommends \
+    clang-tidy-$1 libomp-$1-dev

Exec/Make.Maestro

@@ -56,6 +56,14 @@ endif
 
 include $(AMREX_HOME)/Tools/GNUMake/Make.defs
 
+ifeq ($(USE_GPU),TRUE)
+  # We don't currently support host-side OpenMP being enabled
+  # when using GPUs. Throw an error to prevent this case.
+  ifeq ($(USE_OMP),TRUE)
+    $(error OpenMP is not supported by MAESTROeX when building with GPU support)
+  endif
+endif
+
 MAESTROEX_AUTO_SOURCE_DIR := $(TmpBuildDir)/maestroex_sources/$(optionsSuffix).EXE
 
 all: $(executable)

Exec/science/wdconvect/MaestroInitData.cpp

@@ -4,16 +4,24 @@ using namespace amrex;
 using namespace problem_rp;
 
 // initializes data on a specific level
-void Maestro::InitLevelData(const int lev, const Real time, const MFIter& mfi,
+void Maestro::InitLevelData(const int lev, const Real time, const MFIter& mfi,  // NOLINT(readability-convert-member-functions-to-static)
                             const Array4<Real> scal, const Array4<Real> vel) {
-    Abort("Planar InitLevelData not implemented.");
+    amrex::ignore_unused(lev);
+    amrex::ignore_unused(time);
+    amrex::ignore_unused(mfi);
+    amrex::ignore_unused(scal);
+    amrex::ignore_unused(vel);
+
+    amrex::Abort("Planar InitLevelData not implemented.");
 }
 
 void Maestro::InitLevelDataSphr(const int lev, const Real time, MultiFab& scal,
                                 MultiFab& vel) {
+
+    amrex::ignore_unused(time);
+
     // timer for profiling
     BL_PROFILE_VAR("Maestro::InitLevelDataSphr()", InitLevelDataSphr);
-    const int max_lev = base_geom.max_radial_level + 1;
 
 #ifdef _OPENMP
 #pragma omp parallel
@@ -57,11 +65,11 @@ void Maestro::InitLevelDataSphr(const int lev, const Real time, MultiFab& scal,
         }
     }
 
-    Put1dArrayOnCart(lev, temp_vec, temp_mf, 0, 0, bcs_f, 0);
+    Put1dArrayOnCart(lev, temp_vec, temp_mf, false, false, bcs_f, 0);
     MultiFab::Copy(scal, temp_mf, 0, Temp, 1, 0);
 
     // initialize p0_cart
-    Put1dArrayOnCart(lev, p0_init, p0_cart, 0, 0);
+    Put1dArrayOnCart(lev, p0_init, p0_cart, false, false);
 
     // initialize species
     for (auto comp = 0; comp < NumSpec; ++comp) {
@@ -70,7 +78,7 @@ void Maestro::InitLevelDataSphr(const int lev, const Real time, MultiFab& scal,
                 temp_arr(l, r) = s0_init_arr(l, r, FirstSpec + comp);
             }
         }
-        Put1dArrayOnCart(lev, temp_vec, temp_mf, 0, 0, bcs_s, FirstSpec + comp);
+        Put1dArrayOnCart(lev, temp_vec, temp_mf, false, false, bcs_s, FirstSpec + comp);
         MultiFab::Copy(scal, temp_mf, 0, FirstSpec + comp, 1, 0);
     }
 
@@ -162,7 +170,6 @@ void Maestro::InitLevelDataSphr(const int lev, const Real time, MultiFab& scal,
 
 
         const auto prob_lo = geom[lev].ProbLoArray();
-        const auto prob_hi = geom[lev].ProbHiArray();
         const auto dx = geom[lev].CellSizeArray();
 
         const auto center_p = center;
@@ -179,11 +186,7 @@ void Maestro::InitLevelDataSphr(const int lev, const Real time, MultiFab& scal,
             const Real z = prob_lo[2] + (Real(k) + 0.5) * dx[2] - center_p[2];
 
             // set perturbational velocity to zero
-            Real vpert[3];
-
-            for (auto n = 0; n < 3; ++n) {
-                vpert[n] = 0.0;
-            }
+            Real vpert[3]{};
 
             // compute distance to the center of the star
             Real rloc = std::sqrt(x * x + y * y + z * z);

Exec/science/wdconvect/MaestroTagging.cpp

@@ -4,23 +4,36 @@
 using namespace amrex;
 using namespace problem_rp;
 
-void Maestro::RetagArray(const Box& bx, const int lev) {
+void Maestro::RetagArray(const Box& bx, const int lev) {  // NOLINT(readability-convert-member-functions-to-static)
+
+    amrex::ignore_unused(bx);
+    amrex::ignore_unused(lev);
+
     // timer for profiling
     BL_PROFILE_VAR("Maestro::RetagArray()", RetagArray);
 
     Abort("Error: RetagArray should not be called for spherical");
 }
 
-void Maestro::TagBoxes(TagBoxArray& tags, const MFIter& mfi, const int lev,
+void Maestro::TagBoxes(TagBoxArray& tags, const MFIter& mfi, const int lev,  // NOLINT(readability-convert-member-functions-to-static)
                        const Real time) {
+
+    amrex::ignore_unused(tags);
+    amrex::ignore_unused(mfi);
+    amrex::ignore_unused(lev);
+    amrex::ignore_unused(time);
+
     // timer for profiling
     BL_PROFILE_VAR("Maestro::TagBoxes()", TagBoxes);
 
     Abort("Error: TagBoxes should not be called for spherical");
 }
 
-void Maestro::StateError(TagBoxArray& tags, const MultiFab& state_mf,
+void Maestro::StateError(TagBoxArray& tags, const MultiFab& state_mf,  // NOLINT(readability-convert-member-functions-to-static)
                          const MFIter& mfi, const int lev, const Real time) {
+
+    amrex::ignore_unused(time);
+
     // timer for profiling
     BL_PROFILE_VAR("Maestro::StateError()", StateError);
 

README.md

@@ -178,10 +178,6 @@ status as a core developer.
 
 ## Getting help
 
-Join the mailing list to ask for help or stay up-to-date:
-
-https://groups.google.com/forum/#!forum/maestro-help
-
 For answers to frequently asked questions and solutions to common
 problems, consult the [FAQ
 section](https://amrex-astro.github.io/MAESTROeX/docs/faq.html) of the
@@ -190,3 +186,9 @@ User's Guide.
 If you discover any bugs in the code or would like to request further
 help, then please submit a issue to the MAESTROeX Github repository
 describing your problem.
+
+For general questions, post to the discussion forum:
+https://github.com/AMReX-Astro/MAESTROeX/discussions
+
+and for problems, raise an issue:
+https://github.com/AMReX-Astro/MAESTROeX/issues

Source/Maestro.H

@@ -957,11 +957,12 @@ class Maestro : public amrex::AmrCore {
                            int comp, int bccomp, const bool is_vel,
                            const bool is_conservative) const;
 #else
+    static
     void MakeDivU(const amrex::Box& bx, amrex::Array4<amrex::Real> const divu,
                   amrex::Array4<amrex::Real> const umac,
                   amrex::Array4<amrex::Real> const vmac,
                   amrex::Array4<amrex::Real> const wmac,
-                  const amrex::GpuArray<Real, AMREX_SPACEDIM> dx) const;
+                  const amrex::GpuArray<Real, AMREX_SPACEDIM> dx);
 
     void MakeEdgeScalPredictor(const amrex::MFIter& mfi,
                                amrex::Array4<amrex::Real> const slx,

Source/MaestroDt.cpp

@@ -664,13 +664,14 @@ void Maestro::FirstDt() {
                 Real spdy = spd.max<RunOn::Device>(tileBox, 0) / dx[1];
                 Real pforcey =
                     vel_force[lev][mfi].maxabs<RunOn::Device>(tileBox, 1);
+#if AMREX_SPACEDIM == 2
                 Real spdz = spdy * 0.1;  // for 2d make sure this is < spdy
-                uz /= AMREX_SPACEDIM == 2 ? dx[1] : dx[2];
-#if (AMREX_SPACEDIM == 3)
-                spdz = spd.max<RunOn::Device>(tileBox, 0) / dx[2];
+#else
+                Real spdz = spd.max<RunOn::Device>(tileBox, 0) / dx[2];
                 Real pforcez =
                     vel_force[lev][mfi].maxabs<RunOn::Device>(tileBox, 2);
 #endif
+                uz /= AMREX_SPACEDIM == 2 ? dx[1] : dx[2];
 
                 // use advective constraint unless velocities are zero everywhere
                 // in which case we use the sound speed

Source/MaestroMakeEdgeScalars.cpp

@@ -648,7 +648,7 @@ void Maestro::MakeEdgeScalEdges(
 void Maestro::MakeDivU(const Box& bx, Array4<Real> const divu,
                        Array4<Real> const umac, Array4<Real> const vmac,
                        Array4<Real> const wmac,
-                       const GpuArray<Real, AMREX_SPACEDIM> dx) const {
+                       const GpuArray<Real, AMREX_SPACEDIM> dx) {
     // timer for profiling
     BL_PROFILE_VAR("Maestro::MakeDivU()", MakeDivU);
 

Source/MaestroMakeFlux.cpp

@@ -794,12 +794,11 @@ void Maestro::MakeRhoHFlux(
                         rhoh0mac_edgez.array(mfi);
 
                     ParallelFor(xbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
-                        if (have_h) {
-                            // enthalpy edge state is h
-                            // this is not supported on irregular-spaced base state
-                        } else if (have_hprime) {
-                            // enthalpy edge state is h'
-                            // this is not supported on irregular-spaced base state
+                        if (have_h || have_rhoh) {
+                            // have_h: enthalpy edge state is h
+                            //         this is not supported on irregular-spaced base state
+                            // have_hprime: enthalpy edge state is h'
+                            //              this is not supported on irregular-spaced base state
                         } else if (have_rhoh) {
                             sfluxx(i, j, k, RhoH) =
                                 umacx(i, j, k) * sedgex(i, j, k, RhoH);
@@ -817,12 +816,11 @@ void Maestro::MakeRhoHFlux(
                     });
 
                     ParallelFor(ybx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
-                        if (have_h) {
-                            // enthalpy edge state is h
-                            // this is not supported on irregular-spaced base state
-                        } else if (have_hprime) {
-                            // enthalpy edge state is h'
-                            // this is not supported on irregular-spaced base state
+                        if (have_h || have_hprime) {
+                            // have_h: enthalpy edge state is h
+                            //         this is not supported on irregular-spaced base state
+                            // have_hprime: enthalpy edge state is h'
+                            //              this is not supported on irregular-spaced base state
                         } else if (have_rhoh) {
                             sfluxy(i, j, k, RhoH) =
                                 vmac(i, j, k) * sedgey(i, j, k, RhoH);
@@ -839,12 +837,11 @@ void Maestro::MakeRhoHFlux(
                     });
 
                     ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
-                        if (have_h) {
-                            // enthalpy edge state is h
-                            // this is not supported on irregular-spaced base state
-                        } else if (have_hprime) {
-                            // enthalpy edge state is h'
-                            // this is not supported on irregular-spaced base state
+                        if (have_h || have_hprime) {
+                            // have_h: enthalpy edge state is h
+                            //         this is not supported on irregular-spaced base state
+                            // have_hprime: enthalpy edge state is h'
+                            //              this is not supported on irregular-spaced base state
                         } else if (have_rhoh) {
                             sfluxz(i, j, k, RhoH) =
                                 wmac(i, j, k) * sedgez(i, j, k, RhoH);

Source/MaestroMakeUtrans.cpp

@@ -461,7 +461,7 @@ void Maestro::MakeUtrans(
 
                     // impose hi side bc's
                 } else if (j == domhi[1] + 1) {
-                    switch (bchi) {
+                    switch (bchi) {  // NOLINT(bugprone-switch-missing-default-case)
                         case amrex::PhysBCType::inflow:
                             vly = utilde_arr(i, j + 1, k, 1);
                             vry = utilde_arr(i, j + 1, k, 1);