Adding Waves2AMR library (#898)

* new submodule, which has a new library dependency
* enables use of HOS-Ocean for precursor simulation of waves, then place into AMR-Wind domain and forced
* documentation for ocean waves input arguments

.gitmodules

@@ -7,3 +7,6 @@
 [submodule "submods/AMReX-Hydro"]
 	path = submods/AMReX-Hydro
 	url = https://github.com/AMReX-Fluids/AMReX-Hydro.git
+[submodule "submods/Waves2AMR"]
+	path = submods/Waves2AMR
+	url = https://github.com/mbkuhn/Waves2AMR/

CMakeLists.txt

@@ -35,6 +35,7 @@ set(AMR_WIND_PRECISION "DOUBLE" CACHE STRING "Floating point precision SINGLE or
 # Third party libraries
 option(AMR_WIND_USE_INTERNAL_AMREX "Add AMReX as subproject" ON)
 option(AMR_WIND_USE_INTERNAL_AMREX_HYDRO "Add AMReX-Hydro as subproject" ON)
+option(AMR_WIND_USE_INTERNAL_WAVES2AMR "Add Waves2AMR as subproject" ON)
 option(AMR_WIND_ENABLE_HDF5 "Enable HDF5 library" OFF)
 option(AMR_WIND_ENABLE_HDF5_ZFP "Enable ZFP compression in HDF5 library" OFF)
 option(AMR_WIND_ENABLE_NETCDF "Enable NetCDF library" OFF)
@@ -44,6 +45,7 @@ option(AMR_WIND_ENABLE_HYPRE "Enable HYPRE integration" OFF)
 option(AMR_WIND_ENABLE_OPENFAST "Enable OpenFAST integration" OFF)
 option(AMR_WIND_ENABLE_ASCENT "Enable Ascent visualization library" OFF)
 option(AMR_WIND_ENABLE_UMPIRE "Enable Umpire GPU memory pools" OFF)
+option(AMR_WIND_ENABLE_W2A "Enable Waves2AMR library" OFF)
 
 #Options for C++
 set(CMAKE_CXX_STANDARD 17)
@@ -89,6 +91,12 @@ endif()
 init_amrex()
 init_amrex_hydro()
 
+########################## Waves2AMR ##################################
+if (AMR_WIND_ENABLE_W2A)
+  # submodule for the code, not always compiled
+  init_waves2amr()
+endif()
+
 ########################### AMR-Wind #####################################
 
 # General information about machine, compiler, and build type
@@ -119,6 +127,14 @@ endif()
 
 include(set_compile_flags)
 
+if (AMR_WIND_ENABLE_W2A)
+  # Turn on the use of the library via bool, it has already been included above
+  target_compile_definitions(${amr_wind_lib_name} PUBLIC AMR_WIND_USE_W2A)
+  # Address FFTW dependencies
+  set(CMAKE_PREFIX_PATH ${FFTW_DIR} ${CMAKE_PREFIX_PATH})
+  include_directories(${FFTW_DIR}/include)
+endif()
+
 if(AMR_WIND_ENABLE_NETCDF)
   set(CMAKE_PREFIX_PATH ${NETCDF_DIR} ${CMAKE_PREFIX_PATH})
   find_package(NetCDF REQUIRED)

amr-wind/CMakeLists.txt

@@ -46,6 +46,12 @@ target_link_libraries_system(${amr_wind_lib_name} PUBLIC AMReX::amrex AMReX-Hydr
 target_link_libraries(${amr_wind_exe_name} PRIVATE ${amr_wind_lib_name} AMReX-Hydro::amrex_hydro_api)
 target_link_libraries(${aw_api_lib} PUBLIC ${amr_wind_lib_name} AMReX-Hydro::amrex_hydro_api)
 
+if (AMR_WIND_ENABLE_W2A)
+  target_link_libraries_system(${amr_wind_lib_name} PUBLIC Waves2AMR::waves_2_amr_api)
+  target_link_libraries(${amr_wind_exe_name} PRIVATE ${amr_wind_lib_name} Waves2AMR::waves_2_amr_api)
+  target_link_libraries(${aw_api_lib} PUBLIC ${amr_wind_lib_name} Waves2AMR::waves_2_amr_api)
+endif()
+
 # Set -fpic options
 set_target_properties(${amr_wind_lib_name} buildInfo${amr_wind_lib_name}
   PROPERTIES POSITION_INDEPENDENT_CODE ON)

amr-wind/ocean_waves/OceanWaves.cpp

@@ -62,7 +62,11 @@ void OceanWaves::post_init_actions()
     relaxation_zones();
 }
 
-void OceanWaves::post_regrid_actions() {}
+void OceanWaves::post_regrid_actions()
+{
+    BL_PROFILE("amr-wind::ocean_waves::OceanWaves::post_regrid_actions");
+    m_owm->record_regrid_flag();
+}
 
 void OceanWaves::pre_advance_work()
 {
@@ -83,6 +87,7 @@ void OceanWaves::relaxation_zones()
     BL_PROFILE("amr-wind::ocean_waves::OceanWaves::update_relaxation_zones");
     m_owm->update_relax_zones();
     m_owm->apply_relax_zones();
+    m_owm->reset_regrid_flag();
 }
 
 void OceanWaves::prepare_outputs()

amr-wind/ocean_waves/OceanWavesModel.H

@@ -41,6 +41,10 @@ public:
     virtual void prepare_outputs(const std::string&) = 0;
 
     virtual void write_outputs() = 0;
+
+    virtual void record_regrid_flag() = 0;
+
+    virtual void reset_regrid_flag() = 0;
 };
 
 /** Concrete implementation of the WaveModel for different wave theories.
@@ -102,6 +106,10 @@ public:
         m_out_op.prepare_outputs(out_dir);
     }
 
+    void record_regrid_flag() override { m_data.meta().regrid_occurred = true; }
+
+    void reset_regrid_flag() override { m_data.meta().regrid_occurred = false; }
+
     void write_outputs() override { m_out_op.write_outputs(); }
 
     void init_waves(int level, const amrex::Geometry& geom) override

amr-wind/ocean_waves/relaxation_zones/CMakeLists.txt

@@ -5,4 +5,5 @@ target_sources(${amr_wind_lib_name}
   LinearWaves.cpp
   StokesWaves.cpp
   HOSWaves.cpp
+  W2AWaves.cpp
   )

amr-wind/ocean_waves/relaxation_zones/RelaxationZones.H

@@ -34,6 +34,8 @@ struct RelaxZonesBaseData
     bool has_outprofile{false};
 
     amrex::Real ramp_period{2.0};
+
+    bool regrid_occurred{false};
 };
 
 struct RelaxZonesType : public OceanWavesType

amr-wind/ocean_waves/relaxation_zones/W2AWaves.H

@@ -0,0 +1,85 @@
+#ifndef W2AWAVES_H
+#define W2AWAVES_H
+
+#include "amr-wind/ocean_waves/relaxation_zones/RelaxationZones.H"
+#include <vector>
+#include <complex>
+#ifdef AMR_WIND_USE_W2A
+#include "Waves2AMR.h"
+#endif
+
+namespace amr_wind::ocean_waves {
+
+struct W2AWavesData : public RelaxZonesBaseData
+{
+    // Prefix for HOS files
+    std::string modes_file{"modes_HOS_SWENSE.dat"};
+    // Number of modes in each direction
+    int n0{0};
+    int n1{0};
+    // Spatial resolution in each direction
+    amrex::Real dx0{0.};
+    amrex::Real dx1{0.};
+    // Length to dimensionalize nondim data
+    amrex::Real dimL{0.};
+    // Timestep size of reference data
+    amrex::Real dt_modes{0.0};
+
+    // Time (in HOS sim) to begin reading from
+    // (this corresponds to t = 0 in AMR-Wind sim, functions as offset)
+    amrex::Real t_winit{-1.0};
+    // Timestep (in HOS sim) of latest imported wave data
+    int ntime{0};
+    // Timestep (in HOS sim) of first imported wave data
+    int n_winit{0};
+    // Timestep (in HOS sim) of last available wave data
+    int n_wstop{0};
+    // Timestep offset (in HOS sim) for resetting to earlier data
+    int n_offset{0};
+
+    // Time (in AMR-Wind sim) when ow arrays were last changed
+    // (this is from the last timestep, goes with time-interpolated data)
+    amrex::Real t_last{-1.0};
+    // Time (in AMR-Wind sim) of latest imported wave data
+    amrex::Real t{0.0};
+
+    // Vectors to store modes
+    std::vector<std::complex<double>> mX, mY, mZ, mFS;
+
+// Struct variables that have special types
+#ifdef AMR_WIND_USE_W2A
+    // FFTW plan
+    fftw_plan plan;
+    // FFTW pointers to store modes ready for ifft
+    fftw_complex *eta_mptr, *u_mptr, *v_mptr, *w_mptr;
+    // ReadModes object
+    ReadModes rmodes;
+#endif
+
+    // Height vector (where velocity is sampled) stuff
+    amrex::Vector<amrex::Real> hvec;
+    // Index vector (where hvec overlaps with local boxes)
+    amrex::Vector<int> indvec;
+    // Flag indicating interpolation should take place on this processor
+    bool do_interp{true};
+    // Flag indicating regrid has occurred since the last resizing of vecs
+    bool resize_flag{false};
+    // Vectors for spatial data from transformed modes
+    amrex::Gpu::DeviceVector<amrex::Real> sp_eta_vec, sp_u_vec, sp_v_vec,
+        sp_w_vec;
+    // Spacing of spatial data
+    amrex::Real sp_dx{0.}, sp_dy{0.};
+};
+
+struct W2AWaves : public RelaxZonesType
+{
+    using InfoType = OceanWavesInfo;
+    using MetaType = W2AWavesData;
+    using DataType = OceanWavesDataHolder<W2AWaves>;
+
+    static std::string identifier() { return "W2AWaves"; }
+};
+
+} // namespace amr_wind::ocean_waves
+
+#endif /* W2AWAVES_H */

amr-wind/ocean_waves/relaxation_zones/W2AWaves.cpp

@@ -0,0 +1,9 @@
+#include "amr-wind/ocean_waves/relaxation_zones/W2AWaves.H"
+#include "amr-wind/ocean_waves/relaxation_zones/waves2amr_ops.H"
+#include "amr-wind/ocean_waves/OceanWavesModel.H"
+
+namespace amr_wind::ocean_waves {
+
+template class OWModel<W2AWaves>;
+
+} // namespace amr_wind::ocean_waves

amr-wind/ocean_waves/relaxation_zones/relaxation_zones_ops.cpp

@@ -36,7 +36,7 @@ void read_inputs(
         pp.query("initialize_wave_field", wdata.init_wave_field);
     }
 
-    pp.query("timeramp", wdata.has_ramp);
+    wdata.has_ramp = pp.contains("timeramp_period");
     if (wdata.has_ramp) {
         pp.query("timeramp_period", wdata.ramp_period);
     }